{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "dgBTU57OWYI0"
   },
   "source": [
    "# Install Library\n",
    "\n",
    "[RDKit ](https://github.com/rdkit/rdkit)\n",
    "\n",
    "[DGL](https://github.com/dmlc/dgl/)\n",
    "\n",
    "[DGL-LifeSci](https://github.com/awslabs/dgl-lifesci)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "id": "EOF1QxeqhajG"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install rdkit-pypi\n",
    "!pip install dgllife\n",
    "!pip install --pre dgl-cu113 dglgo -f https://data.dgl.ai/wheels-test/repo.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "xtojkovzWYI2"
   },
   "source": [
    "# Import Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import dgl \n",
    "import sys\n",
    "import random\n",
    "import torch\n",
    "import cv2\n",
    "import torchvision\n",
    "import statistics\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch.optim as optim\n",
    "\n",
    "\n",
    "from rdkit.Chem import AllChem\n",
    "from rdkit import Chem\n",
    "from rdkit import DataStructs\n",
    "\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from dgllife.utils import smiles_to_bigraph, CanonicalAtomFeaturizer, AttentiveFPAtomFeaturizer\n",
    "from tqdm.notebook import tqdm, trange\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from utils.general import DATASET, get_dataset, separate_active_and_inactive_data, get_embedding_vector_class, count_lablel,data_generator\n",
    "from utils.gcn_pre_trained import get_muv_model\n",
    "from model.heterogeneous_siamese_muv import siamese_model_attentiveFp_muv, siamese_model_Canonical_muv\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "device = torch.device('cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "1RVgRpTmQ5rp",
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "cache_path='./muv_dglgraph.bin'\n",
    "\n",
    "df = get_dataset(\"muv\")\n",
    "\n",
    "ids = df['mol_id'].tolist()\n",
    "load_full = False\n",
    "\n",
    "df = df.drop(columns=['mol_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MUV-466</th>\n",
       "      <th>MUV-548</th>\n",
       "      <th>MUV-600</th>\n",
       "      <th>MUV-644</th>\n",
       "      <th>MUV-652</th>\n",
       "      <th>MUV-689</th>\n",
       "      <th>MUV-692</th>\n",
       "      <th>MUV-712</th>\n",
       "      <th>MUV-713</th>\n",
       "      <th>MUV-733</th>\n",
       "      <th>MUV-737</th>\n",
       "      <th>MUV-810</th>\n",
       "      <th>MUV-832</th>\n",
       "      <th>MUV-846</th>\n",
       "      <th>MUV-852</th>\n",
       "      <th>MUV-858</th>\n",
       "      <th>MUV-859</th>\n",
       "      <th>smiles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cc1cccc(N2CCN(C(=O)C34CC5CC(CC(C5)C3)C4)CC2)c1C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Cn1ccnc1SCC(=O)Nc1ccc(Oc2ccccc2)cc1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>COc1cc2c(cc1NC(=O)CN1C(=O)NC3(CCc4ccccc43)C1=O...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O=C1/C(=C/NC2CCS(=O)(=O)C2)c2ccccc2C(=O)N1c1cc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NC(=O)NC(Cc1ccccc1)C(=O)O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93082</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O=C(NCc1ccccc1Cl)C1CCCO1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93083</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COc1cc(NCCCCCN2C(=O)c3ccccc3C2=O)c2ncccc2c1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93084</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CCN(CC)c1ccc(/C=C2/C(=O)ON=C2C)cc1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93085</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cc1cc(=O)oc2cc(OCC(=O)c3ccc4c(c3)NC(=O)CO4)ccc12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93086</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>COc1ccc([N+](=O)[O-])cc1NC(=O)c1ccc(C)o1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>93087 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       MUV-466  MUV-548  MUV-600  MUV-644  MUV-652  MUV-689  MUV-692  MUV-712  \\\n",
       "0          NaN      NaN      NaN      NaN      NaN      NaN      NaN      0.0   \n",
       "1          0.0      0.0      NaN      NaN      0.0      0.0      0.0      NaN   \n",
       "2          NaN      NaN      0.0      NaN      NaN      NaN      NaN      NaN   \n",
       "3          NaN      0.0      0.0      NaN      NaN      0.0      NaN      NaN   \n",
       "4          0.0      NaN      NaN      NaN      0.0      NaN      0.0      0.0   \n",
       "...        ...      ...      ...      ...      ...      ...      ...      ...   \n",
       "93082      NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "93083      NaN      NaN      NaN      NaN      NaN      NaN      NaN      0.0   \n",
       "93084      NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "93085      NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "93086      NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "\n",
       "       MUV-713  MUV-733  MUV-737  MUV-810  MUV-832  MUV-846  MUV-852  MUV-858  \\\n",
       "0          NaN      NaN      NaN      0.0      NaN      NaN      NaN      NaN   \n",
       "1          NaN      NaN      0.0      NaN      0.0      NaN      NaN      0.0   \n",
       "2          NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "3          NaN      NaN      NaN      NaN      NaN      NaN      NaN      0.0   \n",
       "4          NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "...        ...      ...      ...      ...      ...      ...      ...      ...   \n",
       "93082      NaN      NaN      NaN      NaN      NaN      0.0      NaN      NaN   \n",
       "93083      NaN      NaN      NaN      NaN      NaN      NaN      NaN      NaN   \n",
       "93084      NaN      NaN      NaN      NaN      NaN      0.0      NaN      NaN   \n",
       "93085      NaN      NaN      NaN      NaN      NaN      NaN      0.0      NaN   \n",
       "93086      NaN      NaN      NaN      NaN      NaN      NaN      0.0      NaN   \n",
       "\n",
       "       MUV-859                                             smiles  \n",
       "0          NaN    Cc1cccc(N2CCN(C(=O)C34CC5CC(CC(C5)C3)C4)CC2)c1C  \n",
       "1          0.0                Cn1ccnc1SCC(=O)Nc1ccc(Oc2ccccc2)cc1  \n",
       "2          0.0  COc1cc2c(cc1NC(=O)CN1C(=O)NC3(CCc4ccccc43)C1=O...  \n",
       "3          NaN  O=C1/C(=C/NC2CCS(=O)(=O)C2)c2ccccc2C(=O)N1c1cc...  \n",
       "4          NaN                          NC(=O)NC(Cc1ccccc1)C(=O)O  \n",
       "...        ...                                                ...  \n",
       "93082      NaN                           O=C(NCc1ccccc1Cl)C1CCCO1  \n",
       "93083      NaN        COc1cc(NCCCCCN2C(=O)c3ccccc3C2=O)c2ncccc2c1  \n",
       "93084      NaN                 CCN(CC)c1ccc(/C=C2/C(=O)ON=C2C)cc1  \n",
       "93085      NaN   Cc1cc(=O)oc2cc(OCC(=O)c3ccc4c(c3)NC(=O)CO4)ccc12  \n",
       "93086      NaN           COc1ccc([N+](=O)[O-])cc1NC(=O)c1ccc(C)o1  \n",
       "\n",
       "[93087 rows x 18 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true,
    "id": "m_nAHT_WhajK"
   },
   "outputs": [],
   "source": [
    "muv_tasks = df.columns.values[:17].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "hidden": true,
    "id": "GEdp1FUphajL",
    "outputId": "352d95e6-5d5c-4c98-8d51-d7f330c51e9f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MUV-466 one: 27  zero: 78246  NAN: 14814\n",
      "MUV-548 one: 29  zero: 78353  NAN: 14705\n",
      "MUV-600 one: 30  zero: 78359  NAN: 14698\n",
      "MUV-644 one: 30  zero: 78464  NAN: 14593\n",
      "MUV-652 one: 29  zero: 78185  NAN: 14873\n",
      "MUV-689 one: 29  zero: 78486  NAN: 14572\n",
      "MUV-692 one: 30  zero: 78443  NAN: 14614\n",
      "MUV-712 one: 28  zero: 78676  NAN: 14383\n",
      "MUV-713 one: 29  zero: 78251  NAN: 14807\n",
      "MUV-733 one: 28  zero: 78405  NAN: 14654\n",
      "MUV-737 one: 29  zero: 78396  NAN: 14662\n",
      "MUV-810 one: 29  zero: 78443  NAN: 14615\n",
      "MUV-832 one: 30  zero: 78420  NAN: 14637\n",
      "MUV-846 one: 30  zero: 78376  NAN: 14681\n",
      "MUV-852 one: 29  zero: 78436  NAN: 14622\n",
      "MUV-858 one: 29  zero: 78313  NAN: 14745\n",
      "MUV-859 one: 24  zero: 78341  NAN: 14722\n"
     ]
    }
   ],
   "source": [
    "one = []\n",
    "zero = []\n",
    "nan = []\n",
    " \n",
    "for task in muv_tasks:\n",
    "    a = list(df[task].value_counts(dropna=False).to_dict().values())\n",
    "    zero.append(a[0])\n",
    "    nan.append(a[1])\n",
    "    one.append(a[2])\n",
    "    print(task ,\"one:\" ,a[2] ,\" zero:\", a[0], \" NAN:\",a[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(489, 1332593)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(one), sum(zero)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 881
    },
    "hidden": true,
    "id": "QjNMCFSqhajM",
    "outputId": "112c0f92-1e31-467c-e3b5-a9285f9c9f14"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABmsAAATWCAYAAADKEZL3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAACNVElEQVR4nOz9e5jVdb3//z9GziCMIMJInsiQRDRNTVELTAVT0M+28gBS5CGNkk2Jmnu7C63wgJIlZe7CYxi2M9uWSZilbfJMYeIhKxEwQTyMM4oIiOv7hz/XrxFUwIUvldvtutZ1Ne/1Wu/3az2dusbu13utukqlUgkAAAAAAABFbFR6AwAAAAAAABsysQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAADYwl19+eerq6lJXV5dbbrlllecrlUo+8IEPpK6uLoMGDaoef/TRR1NXV5fzzz9/tec9//zzU1dXl0cffTRPPvlk2rZtmyOPPPJ199Hc3JyOHTvmkEMOWev3cMstt7zu/t/MbbfdlvHjx+fZZ59d69euD7/+9a8zfvz4t3yeQYMGrfaf1+WXX/6Wzw0AAKxfYg0AAGygOnfunClTpqxy/NZbb80//vGPdO7ceZ3Pvdlmm+WQQw7JL37xizQ2Nq52zbRp07J06dIce+yx63yddXHbbbflzDPPfEfFmjPPPLPm5918881z++235+CDD675uQEAgNoSawAAYAN1xBFH5Nprr01zc3OL41OmTMmAAQOy1VZbvaXzH3vssVm2bFmmTp262ucvvfTS9OzZU0xYT9q1a5c999wzm2222dt+7RdeeOFtvyYAALybiTUAALCBOuqoo5IkP/nJT6rHmpqacu211+aYY455y+cfMmRItthii1x22WWrPPfggw/mzjvvzGc+85m0bt36Dc/z0EMP5cADD0zHjh3TvXv3nHjiiXnuuedWWXfTTTfl0EMPzRZbbJH27dvnAx/4QE444YQ89dRT1TXjx4/PKaeckiTp3bv3Kh8Hd80112Tw4MHZfPPN06FDh2y//fb56le/miVLlrS41iOPPJIjjzwyvXr1Srt27dKzZ8/st99+mT17dot111xzTQYMGJBOnTpl4403zpAhQ/LnP/+5+vyoUaPyve99L0mqe3n1o+ReT6VSyXnnnZett9467du3z4c//OHceOONq6xb3ceg/f3vf8/nPve59OnTJx07dsz73ve+DBs2LPfdd98qr7///vszePDgdOzYMZtttlm++MUv5oYbbljl4+cGDRqU/v375w9/+EP22muvdOzYsfr7s6bzHDVqVDbeeOM89NBDGTJkSDp16pTNN98855xzTpLkjjvuyD777JNOnTplu+22yxVXXPG68wEAgHejN/63IgAA4D2rS5cu+dSnPpVLL700J5xwQpJXws1GG22UI444IhdeeOFbOv9GG22UUaNG5Zvf/GbuvffefOhDH6o+92rAebMo9MQTT2TgwIFp06ZNvv/976dnz56ZOnVqvvSlL62y9h//+EcGDBiQ4447LvX19Xn00UczadKk7LPPPrnvvvvSpk2bHHfccXnmmWdy0UUX5ec//3k233zzJEm/fv2SJH/7299y0EEHZezYsenUqVMeeuihnHvuubnrrrvyu9/9rnqtgw46KCtXrsx5552XrbbaKk899VRuu+22Fh+tNmHChJxxxhn53Oc+lzPOOCPLly/PxIkT89GPfjR33XVX+vXrl//6r//KkiVL8rOf/Sy333579bWv7mt1zjzzzJx55pk59thj86lPfSoLFizI8ccfn5UrV6Zv375vOM/HH388m266ac4555xsttlmeeaZZ3LFFVdkjz32yJ///Ofq6xcuXJiBAwemU6dOufjii9OjR4/85Cc/We3cX11/9NFH59RTT82ECROy0UYbrdU8k2TFihU57LDDcuKJJ+aUU07J1VdfndNPPz3Nzc259tprc9ppp2WLLbbIRRddlFGjRqV///7Zdddd3/D9AgDAu0YFAADYoFx22WWVJJW777678vvf/76SpDJnzpxKpVKp7L777pVRo0ZVKpVKZYcddqgMHDiw+rq5c+dWklQmTpy42vNOnDixkqQyd+7c6rFHHnmkUldXVxkzZkz12IoVKyoNDQ2Vvffe+033etppp1Xq6uoqs2fPbnH8gAMOqCSp/P73v1/t615++eXKihUrKvPmzaskqfzv//7vG+7zjc5x6623VpJU7r333kqlUqk89dRTlSSVCy+88HVfO3/+/Err1q0rJ510Uovjzz33XKWhoaFy+OGHV4998YtfrKzpv5o1NjZW2rdvX/m3f/u3Fsf/+Mc/VpKs9p/XZZdd9rrne+mllyrLly+v9OnTp/LlL3+5evyUU06p1NXVVe6///4W64cMGbLK3AcOHFhJUrn55pvfcO+vN89KpVL57Gc/W0lSufbaa6vHVqxYUdlss80qSSp/+tOfqseffvrpSqtWrSpf+cpX3vB6AADwbuJj0AAAYAM2cODAbLvttrn00ktz33335e67767JR6C9qnfv3tl3330zderULF++PEly4403ZtGiRWt0nd///vfZYYcdWtyVkyTDhw9fZe3ixYtz4oknZsstt0zr1q3Tpk2bbL311kle+di1NfHII49k+PDhaWhoSKtWrdKmTZsMHDiwxTm6deuWbbfdNhMnTsykSZPy5z//OS+//HKL8/zmN7/JSy+9lM985jN56aWXqo/27dtn4MCBLT5GbG3cfvvtefHFFzNixIgWx/faa6/qe30jL730UiZMmJB+/fqlbdu2ad26ddq2bZu//e1vLWZ06623pn///tU7jl716kfnvVbXrl3z8Y9/fJXjazLPV9XV1eWggw6q/ty6det84AMfyOabb55ddtmlerxbt27p0aNH5s2b96bvFwAA3i18DBoAAGzA6urq8rnPfS7f/e538+KLL2a77bbLRz/60dWuffW7ZVauXLna51966aUkSZs2bVocP/bYYzNixIhcf/31+dSnPpXLLrssG2+8cQ4//PA33d/TTz+d3r17r3K8oaGhxc8vv/xyBg8enMcffzz/9V//lR133DGdOnXKyy+/nD333DNLly5902s9//zz+ehHP5r27dvnm9/8Zrbbbrt07NgxCxYsyGGHHVY9R11dXW6++eacddZZOe+883LyySenW7duGTFiRL71rW+lc+fOeeKJJ5Iku++++2qv9erHhK2tp59+erXv//WOvdZXvvKVfO9738tpp52WgQMHpmvXrtloo41y3HHHtZjR6829Z8+eqz3v6j62bU3n+aqOHTumffv2LY61bds23bp1W+Xcbdu2zYsvvvim7xcAAN4txBoAANjAjRo1Kl/72tfygx/8IN/61rded1337t3TqlWr/POf/1zt8//85z/TqlWrbLrppi2OH3bYYenatWsuvfTSDBw4ML/61a/ymc98JhtvvPGb7m3TTTfNokWLVjn+2mNz5szJvffem8svvzyf/exnq8f//ve/v+k1XvW73/0ujz/+eG655Zbq3R9JWnwPzau23nrrTJkyJUny8MMP56c//WnGjx+f5cuX5wc/+EG6d++eJPnZz362Rne8rKlXZ/t6M9lmm23e8PU//vGP85nPfCYTJkxocfypp57KJpts0uI6rwan115jderq6lY5tjbzBACADZ2PQQMAgA3c+973vpxyyikZNmxYi9DxWu3bt8/ee++d66+/fpW7Gl588cVcf/312WeffVa5O6J9+/YZPnx4ZsyYkXPPPTcrVqxY449a23fffXP//ffn3nvvbXH86quvbvHzq7GgXbt2LY5fcsklq5zz1TWvvbNjbc7xr7bbbrucccYZ2XHHHfOnP/0pSTJkyJC0bt06//jHP7Lbbrut9vFm+1mdPffcM+3bt8/UqVNbHL/tttvW6GPB6urqVnl/N9xwwyoBbuDAgZkzZ04eeOCBFsenTZv2ptf412slaz9PAADYELmzBgAAyDnnnLPG6/bdd98MGDAgY8eOzVZbbZX58+fnwgsvzBNPPPG6/2f+sccem+9973uZNGlSPvjBD2avvfZao+uNHTs2l156aQ4++OB885vfTM+ePTN16tQ89NBDLdZ98IMfzLbbbpuvfvWrqVQq6datW375y1/mpptuWuWcO+64Y5LkO9/5Tj772c+mTZs26du3b/baa6907do1J554Yr7+9a+nTZs2mTp16iqh6C9/+Uu+9KUv5dOf/nT69OmTtm3b5ne/+13+8pe/5Ktf/WqSZJtttslZZ52V//zP/8wjjzySAw88MF27ds0TTzyRu+66K506dcqZZ57ZYj/nnntuPvGJT6RVq1bZaaed0rZt21X23rVr14wbNy7f/OY3c9xxx+XTn/50FixYkPHjx6/Rx6ANHTo0l19+eT74wQ9mp512yqxZszJx4sRsscUWq537Jz7xiZx11lnp2bNnrr766urc1+Rj3NZ0ngAAgDtrAACAtTBgwID88Y9/TO/evTNu3LgccMABGTduXHr37p3bbrstAwYMWO3rdtlll+yyyy6pVCprfFdN8sr3sNx6663p169fvvCFL+Too49O+/btM3ny5Bbr2rRpk1/+8pfZbrvtcsIJJ+Soo47K4sWL89vf/naVcw4aNCinn356fvnLX2afffbJ7rvvnlmzZmXTTTfNDTfckI4dO+boo4/OMccck4033jjXXHPNKnvadttt8/3vfz+f+tSncuihh+aXv/xlLrjggpx11lnVdaeffnp+9rOf5eGHH85nP/vZDBkyJKeeemrmzZuXj33sY9V1w4cPz3HHHZfvf//7GTBgQHbfffc8/vjjrzuTs846K2effXZmzJiRQw45JBdddFF+8IMfpG/fvm86z+985zs5+uijc/bZZ2fYsGG5/vrr8/Of/zzbbrtti3W9evXKrbfemu222y4nnnhiRowYkbZt21bf379+ZNrrWdN5AgAASV2lUqmU3gQAAADvfJ///Ofzk5/8JE8//fRq7/wBAADWjY9BAwAAYBVnnXVWevXqlfe///15/vnn86tf/So/+tGPcsYZZwg1AABQY2INAAAAq2jTpk0mTpyYxx57LC+99FL69OmTSZMm5d///d9Lbw0AAN5zfAwaAAAAAABAQRuV3gAAAAAAAMCGTKwBAAAAAAAoSKwBAAAAAAAoqHXpDbyXvPzyy3n88cfTuXPn1NXVld4OAAAAAABQUKVSyXPPPZdevXplo41e//4ZsaaGHn/88Wy55ZaltwEAAAAAALyDLFiwIFtsscXrPi/W1FDnzp2TvDL0Ll26FN4NAAAAAABQUnNzc7bccstqP3g9Yk0NvfrRZ126dBFrAAAAAACAJHnTr055/Q9IAwAAAAAAYL0TawAAAAAAAAoSawAAAAAAAArynTUAAAAAAPAOt3LlyqxYsaL0NniNVq1apXXr1m/6nTRvRqwBAAAAAIB3sOeffz6PPfZYKpVK6a2wGh07dszmm2+etm3brvM5xBoAAAAAAHiHWrlyZR577LF07Ngxm2222Vu+g4PaqVQqWb58eZ588snMnTs3ffr0yUYbrdu3z4g1AAAAAADwDrVixYpUKpVsttlm6dChQ+nt8BodOnRImzZtMm/evCxfvjzt27dfp/OsW+IBAAAAAADeNu6oeeda17tpWpyjBvsAAAAAAABgHYk1AAAAAAAABfnOGgAAAAAAeJepO/Pt/Vi0ytcrb+v1Vufyyy/P2LFj8+yzz5beSs25swYAAAAAAFgvbrvttrRq1SoHHnjgWr1um222yYUXXtji2BFHHJGHH364hrt75xBrAAAAAACA9eLSSy/NSSedlJkzZ2b+/Plv6VwdOnRIjx49arSzdxaxBgAAAAAAqLklS5bkpz/9ab7whS9k6NChufzyy1s8f/3112e33XZL+/bt07179xx22GFJkkGDBmXevHn58pe/nLq6utTVvfKRb5dffnk22WSTJMlf//rX1NXV5aGHHmpxzkmTJmWbbbZJpfLKx7Y98MADOeigg7LxxhunZ8+eGTlyZJ566qn1+8bXgVgDAAAAAADU3DXXXJO+ffumb9++Ofroo3PZZZdVI8oNN9yQww47LAcffHD+/Oc/5+abb85uu+2WJPn5z3+eLbbYImeddVYWLlyYhQsXrnLuvn37Ztddd83UqVNbHL/66qszfPjw1NXVZeHChRk4cGB23nnn3HPPPZk+fXqeeOKJHH744ev/za+l1qU3AAAAAAAAvPdMmTIlRx99dJLkwAMPzPPPP5+bb745+++/f771rW/lyCOPzJlnnlld/6EPfShJ0q1bt7Rq1SqdO3dOQ0PD655/xIgRmTx5cr7xjW8kSR5++OHMmjUrV155ZZLk4osvzoc//OFMmDCh+ppLL700W265ZR5++OFst912NX/P68qdNQAAAAAAQE399a9/zV133ZUjjzwySdK6descccQRufTSS5Mks2fPzn777feWrnHkkUdm3rx5ueOOO5IkU6dOzc4775x+/folSWbNmpXf//732XjjjauPD37wg0mSf/zjH2/p2rXmzhoAAAAAAKCmpkyZkpdeeinve9/7qscqlUratGmTxsbGdOjQ4S1fY/PNN8++++6bq6++OnvuuWd+8pOf5IQTTqg+//LLL2fYsGE599xzV/vadxKxBgAAAAAAqJmXXnopV155ZS644IIMHjy4xXOf/OQnM3Xq1Oy00065+eab87nPfW6152jbtm1Wrlz5ptcaMWJETjvttBx11FH5xz/+Ub2TJ0k+/OEP59prr80222yT1q3f2TnEx6ABAAAAAAA186tf/SqNjY059thj079//xaPT33qU5kyZUq+/vWv5yc/+Um+/vWv58EHH8x9992X8847r3qObbbZJn/4wx/yz3/+M0899dTrXuuwww5Lc3NzvvCFL2TfffdtcSfPF7/4xTzzzDM56qijctddd+WRRx7JjBkzcswxx6xRCHo7vbNTEgAAAAAAsIrK1yult/C6pkyZkv333z/19fWrPPfJT34yEyZMSJcuXfI///M/+cY3vpFzzjknXbp0ycc+9rHqurPOOisnnHBCtt122yxbtiyVyurfb5cuXTJs2LD8z//8T/X7cF7Vq1ev/PGPf8xpp52WIUOGZNmyZdl6661z4IEHZqON3ln3stRVXu8dstaam5tTX1+fpqamdOnSpfR2AAAAAAB4l3vxxRczd+7c9O7dO+3bty+9HVbjjf4ZrWk3eGelIwAAAAAAgA2MWAMAAAAAAFCQWAMAAAAAAFCQWAMAAAAAAFCQWAMAAAAAAFCQWAMAAAAAAFCQWAMAAAAAAFBQ0Vjz0ksv5Ywzzkjv3r3ToUOHvP/9789ZZ52Vl19+ubqmUqlk/Pjx6dWrVzp06JBBgwbl/vvvb3GeZcuW5aSTTkr37t3TqVOnHHLIIXnsscdarGlsbMzIkSNTX1+f+vr6jBw5Ms8++2yLNfPnz8+wYcPSqVOndO/ePWPGjMny5cvX2/sHAAAAAAAoGmvOPffc/OAHP8jkyZPz4IMP5rzzzsvEiRNz0UUXVdecd955mTRpUiZPnpy77747DQ0NOeCAA/Lcc89V14wdOzbXXXddpk2blpkzZ+b555/P0KFDs3Llyuqa4cOHZ/bs2Zk+fXqmT5+e2bNnZ+TIkdXnV65cmYMPPjhLlizJzJkzM23atFx77bU5+eST355hAAAAAAAAG6S6SqVSKXXxoUOHpmfPnpkyZUr12Cc/+cl07NgxV111VSqVSnr16pWxY8fmtNNOS/LKXTQ9e/bMueeemxNOOCFNTU3ZbLPNctVVV+WII45Ikjz++OPZcsst8+tf/zpDhgzJgw8+mH79+uWOO+7IHnvskSS54447MmDAgDz00EPp27dvbrzxxgwdOjQLFixIr169kiTTpk3LqFGjsnjx4nTp0uVN309zc3Pq6+vT1NS0RusBAAAAAOCNvPjii5k7d2569+6d9u3bV4/X1b29+yhXEtbdLbfckn333TeNjY3ZZJNN1tt1Xu+fUbLm3aDonTX77LNPbr755jz88MNJknvvvTczZ87MQQcdlCSZO3duFi1alMGDB1df065duwwcODC33XZbkmTWrFlZsWJFizW9evVK//79q2tuv/321NfXV0NNkuy5556pr69vsaZ///7VUJMkQ4YMybJlyzJr1qzV7n/ZsmVpbm5u8QAAAAAAgA3dqFGj8v/+3/972643aNCgjB07tsWxvfbaKwsXLkx9ff3bto911brkxU877bQ0NTXlgx/8YFq1apWVK1fmW9/6Vo466qgkyaJFi5IkPXv2bPG6nj17Zt68edU1bdu2TdeuXVdZ8+rrFy1alB49eqxy/R49erRY89rrdO3aNW3btq2uea2zzz47Z5555tq+bQAAAAAAYD1r27ZtGhoaSm9jjRS9s+aaa67Jj3/841x99dX505/+lCuuuCLnn39+rrjiihbr6l5zP1elUlnl2Gu9ds3q1q/Lmn91+umnp6mpqfpYsGDBG+4JAAAAAAA2NIMGDcqYMWNy6qmnplu3bmloaMj48eNbrJk0aVJ23HHHdOrUKVtuuWVGjx6d559/vsWaP/7xjxk4cGA6duyYrl27ZsiQIWlsbMyoUaNy66235jvf+U7q6upSV1eXRx99NLfcckvq6ury7LPPpqmpKR06dMj06dNbnPPnP/95OnXqVL3WP//5zxxxxBHp2rVrNt100xx66KF59NFH1+d4khSONaecckq++tWv5sgjj8yOO+6YkSNH5stf/nLOPvvsJKkWr9fe2bJ48eLqXTANDQ1Zvnx5Ghsb33DNE088scr1n3zyyRZrXnudxsbGrFixYpU7bl7Vrl27dOnSpcUDAAAAAABo6YorrkinTp1y55135rzzzstZZ52Vm266qfr8RhttlO9+97uZM2dOrrjiivzud7/LqaeeWn1+9uzZ2W+//bLDDjvk9ttvz8yZMzNs2LCsXLky3/nOdzJgwIAcf/zxWbhwYRYuXJgtt9yyxfXr6+tz8MEHZ+rUqS2OX3311Tn00EOz8cYb54UXXsi+++6bjTfeOH/4wx8yc+bMbLzxxjnwwAOzfPny9TqforHmhRdeyEYbtdxCq1at8vLLLydJevfunYaGhhb/wJYvX55bb701e+21V5Jk1113TZs2bVqsWbhwYebMmVNdM2DAgDQ1NeWuu+6qrrnzzjvT1NTUYs2cOXOycOHC6poZM2akXbt22XXXXWv8zgEAAAAAYMOx00475etf/3r69OmTz3zmM9ltt91y8803V58fO3Zs9t133/Tu3Tsf//jH841vfCM//elPq8+fd9552W233fL9738/H/rQh7LDDjvkS1/6Urp37576+vq0bds2HTt2TENDQxoaGtKqVatV9jBixIj84he/yAsvvJAkaW5uzg033JCjjz46STJt2rRstNFG+dGPfpQdd9wx22+/fS677LLMnz8/t9xyy3qdT9HvrBk2bFi+9a1vZauttsoOO+yQP//5z5k0aVKOOeaYJK98LNnYsWMzYcKE9OnTJ3369MmECRPSsWPHDB8+PMkrNezYY4/NySefnE033TTdunXLuHHjsuOOO2b//fdPkmy//fY58MADc/zxx+eSSy5Jknz+85/P0KFD07dv3yTJ4MGD069fv4wcOTITJ07MM888k3HjxuX44493xwwAAAAAALwFO+20U4ufN9988yxevLj68+9///tMmDAhDzzwQJqbm/PSSy/lxRdfzJIlS9KpU6fMnj07n/70p9/SHg4++OC0bt06119/fY488shce+216dy5cwYPHpwkmTVrVv7+97+nc+fOLV734osv5h//+MdbuvabKRprLrroovzXf/1XRo8encWLF6dXr1454YQT8rWvfa265tRTT83SpUszevToNDY2Zo899siMGTNaDOvb3/52WrduncMPPzxLly7Nfvvtl8svv7xFOZs6dWrGjBlTHfohhxySyZMnV59v1apVbrjhhowePTp77713OnTokOHDh+f8889/GyYBAAAAAADvXW3atGnxc11dXfVTtubNm5eDDjooJ554Yr7xjW+kW7dumTlzZo499tisWLEiSdKhQ4e3vIe2bdvmU5/6VK6++uoceeSRufrqq3PEEUekdetXUsnLL7+cXXfddZWPSkuSzTbb7C1f/40UjTWdO3fOhRdemAsvvPB119TV1WX8+PGrfNnQv2rfvn0uuuiiXHTRRa+7plu3bvnxj3/8hvvZaqut8qtf/erNtg0AAAAAANTIPffck5deeikXXHBB9atT/vUj0JJX7sy5+eabc+aZZ672HG3bts3KlSvf9FojRozI4MGDc//99+f3v/99vvGNb1Sf+/CHP5xrrrkmPXr0eNs/cavod9YAAAAAAAAbtm233TYvvfRSLrroojzyyCO56qqr8oMf/KDFmtNPPz133313Ro8enb/85S956KGHcvHFF+epp55KkmyzzTa588478+ijj+app56q3rXzWgMHDkzPnj0zYsSIbLPNNtlzzz2rz40YMSLdu3fPoYcemv/7v//L3Llzc+utt+bf//3f89hjj62/AUSsAQAAAACAd51K5e19rE8777xzJk2alHPPPTf9+/fP1KlTc/bZZ7dYs91222XGjBm5995785GPfCQDBgzI//7v/1Y/wmzcuHFp1apV+vXrl8022yzz589f7bXq6upy1FFH5d57782IESNaPNexY8f84Q9/yFZbbZXDDjss22+/fY455pgsXbp0vd9pU1eprO8xbziam5tTX1+fpqamt/0WKQAAAAAA3ntefPHFzJ07N71790779u1Lb4fVeKN/RmvaDdxZAwAAAAAAUJBYAwAAAAAAUJBYAwAAAAAAUJBYAwAAAAAAUJBYAwAAAAAAUJBYAwAAAAAAUJBYAwAAAAAAUJBYAwAAAAAAUJBYAwAAAAAAUFDr0hsAAADWj7q69XPeSmX9nBcAAFgL6+sP/tfjXwTWK3fWAAAAAAAANTVq1KjU1dXlnHPOaXH8F7/4RepWE5r69u2btm3b5p///Ocqzw0aNCh1dXWZNm1ai+MXXnhhttlmm5ruuxSxBgBgHdXV1f4BAAAA7xXt27fPueeem8bGxjdcN3PmzLz44ov59Kc/ncsvv/x1z3XGGWdkxYoV62Gn5Yk1ALxj+T/CYcPjv/ew4fHf+9oyz9oz09ozU97p/I5C7ey///5paGjI2Wef/YbrpkyZkuHDh2fkyJG59NJLU1nNR64dddRRaWpqyg9/+MP1td2ixBrYgPnjo7bMEwAAgLebfxeFDc8997zyeDdo1apVJkyYkIsuuiiPPfbYatc899xz+Z//+Z8cffTROeCAA7JkyZLccsstq6zr0qVL/uM//iNnnXVWlixZsp53/vYTawAAAAAAWC/WR1AUFd9d/u3f/i0777xzvv71r6/2+WnTpqVPnz7ZYYcd0qpVqxx55JGZMmXKateOHj067du3z6RJk9bnlosQa3jX8D/qAG+d/y0FeGv87ygAAKy9c889N1dccUUeeOCBVZ6bMmVKjj766OrPRx99dH7+85/n2WefXWVtu3btctZZZ2XixIl56qmn1ueW33ZiDQAAAAAAsN587GMfy5AhQ/If//EfLY4/8MADufPOO3PqqaemdevWad26dfbcc88sXbo0P/nJT1Z7rqOPPjrbbLNNvvnNb74dW3/btC69AQAAAAAA4L3tnHPOyc4775ztttuuemzKlCn52Mc+lu9973st1l511VWZMmVKvvCFL6xyno022ihnn312DjvssNU+/27lzhoAAAAAAGC92nHHHTNixIhcdNFFSZIVK1bkqquuylFHHZX+/fu3eBx33HGZNWtW7r333tWe6+CDD84ee+yRSy655O18C+uVO2sAAAAAAOBd5p67KzU712671exUb+gb3/hGfvrTnyZJrr/++jz99NP5t3/7t1XW9enTJzvuuGOmTJmS7373u6s917nnnpu99tprve737STWAAAAAAAANXX55ZevcmzrrbfOiy++WP155cqVr/v6v/zlL9X/fMstt6zy/IABA1Kp1C5YleZj0AAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAA4B2uUqmU3gKvoxb/bMQaAAAAAAB4h2rVqlWSZPny5YV3wut54YUXkiRt2rRZ53O0rtVmAAAAAACA2mrdunU6duyYJ5988v8XA2p/D8aLL9b8lBuESqWSF154IYsXL84mm2xSDWvrQqwBAAAAAIB3qLq6umy++eaZO3du5s2bl6eeqv015s6t/Tk3JJtsskkaGhre0jnEGgAAAAAAeAdr27Zt+vTpk+XLl+cTn6j9+R96qPbn3FC0adPmLd1R8yqxBgAAAAAA3uE22mijtG/fPvPm1f7c7dvX/pysndp/uB0AAAAAAABrTKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoSKwBAAAAAAAoqGis2WabbVJXV7fK44tf/GKSpFKpZPz48enVq1c6dOiQQYMG5f77729xjmXLluWkk05K9+7d06lTpxxyyCF57LHHWqxpbGzMyJEjU19fn/r6+owcOTLPPvtsizXz58/PsGHD0qlTp3Tv3j1jxozJ8uXL1+v7BwAAAAAAKBpr7r777ixcuLD6uOmmm5Ikn/70p5Mk5513XiZNmpTJkyfn7rvvTkNDQw444IA899xz1XOMHTs21113XaZNm5aZM2fm+eefz9ChQ7Ny5crqmuHDh2f27NmZPn16pk+fntmzZ2fkyJHV51euXJmDDz44S5YsycyZMzNt2rRce+21Ofnkk9+mSQAAAAAAABuqukqlUim9iVeNHTs2v/rVr/K3v/0tSdKrV6+MHTs2p512WpJX7qLp2bNnzj333JxwwglpamrKZpttlquuuipHHHFEkuTxxx/PlltumV//+tcZMmRIHnzwwfTr1y933HFH9thjjyTJHXfckQEDBuShhx5K3759c+ONN2bo0KFZsGBBevXqlSSZNm1aRo0alcWLF6dLly5rtP/m5ubU19enqalpjV/Dmqurq/053zm//WWYaW2ZZ+2Zae2ZaW2ZZ+2ZaW2tj3kmZlprG/I8EzOtNfOsPTOtPTOtLfOsPTOtLX+T1p7f0XeXNe0G75jvrFm+fHl+/OMf55hjjkldXV3mzp2bRYsWZfDgwdU17dq1y8CBA3PbbbclSWbNmpUVK1a0WNOrV6/079+/uub2229PfX19NdQkyZ577pn6+voWa/r3718NNUkyZMiQLFu2LLNmzXrdPS9btizNzc0tHgAAAAAAAGvjHRNrfvGLX+TZZ5/NqFGjkiSLFi1KkvTs2bPFup49e1afW7RoUdq2bZuuXbu+4ZoePXqscr0ePXq0WPPa63Tt2jVt27atrlmds88+u/o9OPX19dlyyy3X4h0DAAAAAAC8g2LNlClT8olPfKLF3S1JUveae7oqlcoqx17rtWtWt35d1rzW6aefnqampupjwYIFb7gvAAAAAACA13pHxJp58+blt7/9bY477rjqsYaGhiRZ5c6WxYsXV++CaWhoyPLly9PY2PiGa5544olVrvnkk0+2WPPa6zQ2NmbFihWr3HHzr9q1a5cuXbq0eAAAAAAAAKyNd0Ssueyyy9KjR48cfPDB1WO9e/dOQ0NDbrrppuqx5cuX59Zbb81ee+2VJNl1113Tpk2bFmsWLlyYOXPmVNcMGDAgTU1Nueuuu6pr7rzzzjQ1NbVYM2fOnCxcuLC6ZsaMGWnXrl123XXX9fOmAQAAAAAAkrQuvYGXX345l112WT772c+mdev//3bq6uoyduzYTJgwIX369EmfPn0yYcKEdOzYMcOHD0+S1NfX59hjj83JJ5+cTTfdNN26dcu4ceOy4447Zv/990+SbL/99jnwwANz/PHH55JLLkmSfP7zn8/QoUPTt2/fJMngwYPTr1+/jBw5MhMnTswzzzyTcePG5fjjj3e3DAAAAAAAsF4VjzW//e1vM3/+/BxzzDGrPHfqqadm6dKlGT16dBobG7PHHntkxowZ6dy5c3XNt7/97bRu3TqHH354li5dmv322y+XX355WrVqVV0zderUjBkzJoMHD06SHHLIIZk8eXL1+VatWuWGG27I6NGjs/fee6dDhw4ZPnx4zj///PX4zgEAAAAAAJK6SqVSKb2J94rm5ubU19enqanJHTnrQV1d7c+5of/2m2ltmWftmWntmWltmWftmWltrY95JmZaaxvyPBMzrTXzrD0zrT0zrS3zrD0zrS1/k9ae39F3lzXtBu+I76wBAAAAAADYUIk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABYk1AAAAAAAABRWPNf/85z9z9NFHZ9NNN03Hjh2z8847Z9asWdXnK5VKxo8fn169eqVDhw4ZNGhQ7r///hbnWLZsWU466aR07949nTp1yiGHHJLHHnusxZrGxsaMHDky9fX1qa+vz8iRI/Pss8+2WDN//vwMGzYsnTp1Svfu3TNmzJgsX758vb13AAAAAACAorGmsbExe++9d9q0aZMbb7wxDzzwQC644IJssskm1TXnnXdeJk2alMmTJ+fuu+9OQ0NDDjjggDz33HPVNWPHjs11112XadOmZebMmXn++eczdOjQrFy5srpm+PDhmT17dqZPn57p06dn9uzZGTlyZPX5lStX5uCDD86SJUsyc+bMTJs2Lddee21OPvnkt2UWAAAAAADAhqmuUqlUSl38q1/9av74xz/m//7v/1b7fKVSSa9evTJ27NicdtppSV65i6Znz54599xzc8IJJ6SpqSmbbbZZrrrqqhxxxBFJkscffzxbbrllfv3rX2fIkCF58MEH069fv9xxxx3ZY489kiR33HFHBgwYkIceeih9+/bNjTfemKFDh2bBggXp1atXkmTatGkZNWpUFi9enC5durzp+2lubk59fX2amprWaD1rp66u9ucs99v/zmCmtWWetWemtWemtWWetWemtbU+5pmYaa1tyPNMzLTWzLP2zLT2zLS2zLP2zLS2/E1ae35H313WtBsUvbPm+uuvz2677ZZPf/rT6dGjR3bZZZf88Ic/rD4/d+7cLFq0KIMHD64ea9euXQYOHJjbbrstSTJr1qysWLGixZpevXqlf//+1TW333576uvrq6EmSfbcc8/U19e3WNO/f/9qqEmSIUOGZNmyZS0+lu1fLVu2LM3NzS0eAAAAAAAAa6NorHnkkUdy8cUXp0+fPvnNb36TE088MWPGjMmVV16ZJFm0aFGSpGfPni1e17Nnz+pzixYtStu2bdO1a9c3XNOjR49Vrt+jR48Wa157na5du6Zt27bVNa919tlnV78Dp76+PltuueXajgAAAAAAANjAFY01L7/8cj784Q9nwoQJ2WWXXXLCCSfk+OOPz8UXX9xiXd1r7uuqVCqrHHut165Z3fp1WfOvTj/99DQ1NVUfCxYseMM9AQAAAAAAvFbRWLP55punX79+LY5tv/32mT9/fpKkoaEhSVa5s2Xx4sXVu2AaGhqyfPnyNDY2vuGaJ554YpXrP/nkky3WvPY6jY2NWbFixSp33LyqXbt26dKlS4sHAAAAAADA2igaa/bee+/89a9/bXHs4YcfztZbb50k6d27dxoaGnLTTTdVn1++fHluvfXW7LXXXkmSXXfdNW3atGmxZuHChZkzZ051zYABA9LU1JS77rqruubOO+9MU1NTizVz5szJwoULq2tmzJiRdu3aZdddd63xOwcAAAAAAHhF65IX//KXv5y99torEyZMyOGHH5677ror//3f/53//u//TvLKx5KNHTs2EyZMSJ8+fdKnT59MmDAhHTt2zPDhw5Mk9fX1OfbYY3PyySdn0003Tbdu3TJu3LjsuOOO2X///ZO8crfOgQcemOOPPz6XXHJJkuTzn/98hg4dmr59+yZJBg8enH79+mXkyJGZOHFinnnmmYwbNy7HH3+8O2YAAAAAAID1pmis2X333XPdddfl9NNPz1lnnZXevXvnwgsvzIgRI6prTj311CxdujSjR49OY2Nj9thjj8yYMSOdO3eurvn2t7+d1q1b5/DDD8/SpUuz33775fLLL0+rVq2qa6ZOnZoxY8Zk8ODBSZJDDjkkkydPrj7fqlWr3HDDDRk9enT23nvvdOjQIcOHD8/555//NkwCAAAAAADYUNVVKpVK6U28VzQ3N6e+vj5NTU3uxlkP6upqf84N/bffTGvLPGvPTGvPTGvLPGvPTGtrfcwzMdNa25DnmZhprZln7Zlp7ZlpbZln7ZlpbfmbtPb8jr67rGk3KPqdNQAAAAAAABs6sQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKAgsQYAAAAAAKCgorFm/Pjxqaura/FoaGioPl+pVDJ+/Pj06tUrHTp0yKBBg3L//fe3OMeyZcty0kknpXv37unUqVMOOeSQPPbYYy3WNDY2ZuTIkamvr099fX1GjhyZZ599tsWa+fPnZ9iwYenUqVO6d++eMWPGZPny5evtvQMAAAAAACTvgDtrdthhhyxcuLD6uO+++6rPnXfeeZk0aVImT56cu+++Ow0NDTnggAPy3HPPVdeMHTs21113XaZNm5aZM2fm+eefz9ChQ7Ny5crqmuHDh2f27NmZPn16pk+fntmzZ2fkyJHV51euXJmDDz44S5YsycyZMzNt2rRce+21Ofnkk9+eIQAAAAAAABusukqlUil18fHjx+cXv/hFZs+evcpzlUolvXr1ytixY3PaaacleeUump49e+bcc8/NCSeckKampmy22Wa56qqrcsQRRyRJHn/88Wy55Zb59a9/nSFDhuTBBx9Mv379cscdd2SPPfZIktxxxx0ZMGBAHnroofTt2zc33nhjhg4dmgULFqRXr15JkmnTpmXUqFFZvHhxunTpskbvp7m5OfX19Wlqalrj17Dm6upqf85yv/3vDGZaW+ZZe2Zae2ZaW+ZZe2ZaW+tjnomZ1tqGPM/ETGvNPGvPTGvPTGvLPGvPTGvL36S153f03WVNu0HxO2v+9re/pVevXundu3eOPPLIPPLII0mSuXPnZtGiRRk8eHB1bbt27TJw4MDcdtttSZJZs2ZlxYoVLdb06tUr/fv3r665/fbbU19fXw01SbLnnnumvr6+xZr+/ftXQ02SDBkyJMuWLcusWbNed+/Lli1Lc3NziwcAAAAAAMDaKBpr9thjj1x55ZX5zW9+kx/+8IdZtGhR9tprrzz99NNZtGhRkqRnz54tXtOzZ8/qc4sWLUrbtm3TtWvXN1zTo0ePVa7do0ePFmtee52uXbumbdu21TWrc/bZZ1e/B6e+vj5bbrnlWk4AAAAAAADY0BWNNZ/4xCfyyU9+MjvuuGP233//3HDDDUmSK664orqm7jX3dFUqlVWOvdZr16xu/bqsea3TTz89TU1N1ceCBQvecF8AAAAAAACvVfxj0P5Vp06dsuOOO+Zvf/tbGhoakmSVO1sWL15cvQumoaEhy5cvT2Nj4xuueeKJJ1a51pNPPtlizWuv09jYmBUrVqxyx82/ateuXbp06dLiAQAAAAAAsDbeUbFm2bJlefDBB7P55pund+/eaWhoyE033VR9fvny5bn11luz1157JUl23XXXtGnTpsWahQsXZs6cOdU1AwYMSFNTU+66667qmjvvvDNNTU0t1syZMycLFy6srpkxY0batWuXXXfddb2+ZwAAAAAAYMPWuuTFx40bl2HDhmWrrbbK4sWL881vfjPNzc357Gc/m7q6uowdOzYTJkxInz590qdPn0yYMCEdO3bM8OHDkyT19fU59thjc/LJJ2fTTTdNt27dMm7cuOrHqiXJ9ttvnwMPPDDHH398LrnkkiTJ5z//+QwdOjR9+/ZNkgwePDj9+vXLyJEjM3HixDzzzDMZN25cjj/+eHfLAAAAAAAA61XRWPPYY4/lqKOOylNPPZXNNtsse+65Z+64445svfXWSZJTTz01S5cuzejRo9PY2Jg99tgjM2bMSOfOnavn+Pa3v53WrVvn8MMPz9KlS7Pffvvl8ssvT6tWraprpk6dmjFjxmTw4MFJkkMOOSSTJ0+uPt+qVavccMMNGT16dPbee+906NAhw4cPz/nnn/82TQIAAAAAANhQ1VUqlUrpTbxXNDc3p76+Pk1NTe7IWQ/q6mp/zg39t99Ma8s8a89Ma89Ma8s8a89Ma2t9zDMx01rbkOeZmGmtmWftmWntmWltmWftmWlt+Zu09vyOvrusaTd4R31nDQAAAAAAwIZGrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChIrAEAAAAAAChonWLN0qVL88ILL1R/njdvXi688MLMmDGjZhsDAAAAAADYEKxTrDn00ENz5ZVXJkmeffbZ7LHHHrngggty6KGH5uKLL67pBgEAAAAAAN7L1inW/OlPf8pHP/rRJMnPfvaz9OzZM/PmzcuVV16Z7373uzXdIAAAAAAAwHvZOsWaF154IZ07d06SzJgxI4cddlg22mij7Lnnnpk3b15NNwgAAAAAAPBetk6x5gMf+EB+8YtfZMGCBfnNb36TwYMHJ0kWL16cLl261HSDAAAAAAAA72XrFGu+9rWvZdy4cdlmm23ykY98JAMGDEjyyl02u+yyS003CAAAAAAA8F5WV6lUKuvywkWLFmXhwoX50Ic+lI02eqX53HXXXenSpUs++MEP1nST7xbNzc2pr69PU1OTO4zWg7q62p9z3X773zvMtLbMs/bMtPbMtLbMs/bMtLbWxzwTM621DXmeiZnWmnnWnpnWnpnWlnnWnpnWlr9Ja8/v6LvLmnaDdbqzJkkaGhrSuXPn3HTTTVm6dGmSZPfdd99gQw0AAAAAAMC6WKdY8/TTT2e//fbLdtttl4MOOigLFy5Mkhx33HE5+eSTa7pBAAAAAACA97J1ijVf/vKX06ZNm8yfPz8dO3asHj/iiCMyffr0mm0OAAAAAADgva71urxoxowZ+c1vfpMtttiixfE+ffpk3rx5NdkYAAAAAADAhmCd7qxZsmRJiztqXvXUU0+lXbt2b3lTAAAAAAAAG4p1ijUf+9jHcuWVV1Z/rqury8svv5yJEydm3333rdnmAAAAAAAA3uvW6WPQJk6cmEGDBuWee+7J8uXLc+qpp+b+++/PM888kz/+8Y+13iMAAAAAAMB71jrdWdOvX7/85S9/yUc+8pEccMABWbJkSQ477LD8+c9/zrbbblvrPQIAAAAAALxn1VUqlUrpTbxXNDc3p76+Pk1NTenSpUvp7bzn1NXV/pwb+m+/mdaWedaemdaemdaWedaemdbW+phnYqa1tiHPMzHTWjPP2jPT2jPT2jLP2jPT2vI3ae35HX13WdNusE531kyfPj0zZ86s/vy9730vO++8c4YPH57GxsZ1OSUAAAAAAMAGaZ1izSmnnJLm5uYkyX333ZevfOUrOeigg/LII4/kK1/5Sk03CAAAAAAA8F7Wel1eNHfu3PTr1y9Jcu2112bYsGGZMGFC/vSnP+Wggw6q6QYBAAAAAADey9bpzpq2bdvmhRdeSJL89re/zeDBg5Mk3bp1q95xAwAAAAAAwJtbpztr9tlnn3zlK1/J3nvvnbvuuivXXHNNkuThhx/OFltsUdMNAgAAAAAAvJet0501kydPTuvWrfOzn/0sF198cd73vvclSW688cYceOCBNd0gAAAAAADAe1ldpVKplN7Ee0Vzc3Pq6+vT1NSULl26lN7Oe05dXe3PuaH/9ptpbZln7Zlp7ZlpbZln7Zlpba2PeSZmWmsb8jwTM60186w9M609M60t86w9M60tf5PWnt/Rd5c17Qbr9DFo/2rp0qVZsWJFi2NCBQAAAAAAwJpZp49BW7JkSb70pS+lR48e2XjjjdO1a9cWDwAAAAAAANbMOsWaU089Nb/73e/y/e9/P+3atcuPfvSjnHnmmenVq1euvPLKWu8RAAAAAADgPWudPgbtl7/8Za688soMGjQoxxxzTD760Y/mAx/4QLbeeutMnTo1I0aMqPU+AQAAAAAA3pPW6c6aZ555Jr17907yyvfTPPPMM0mSffbZJ3/4wx9qtzsAAAAAAID3uHWKNe9///vz6KOPJkn69euXn/70p0leueNmk002qdXeAAAAAAAA3vPWKdZ87nOfy7333pskOf3006vfXTN27NiccsopNd0gAAAAAADAe1ldpVKpvNWTzJ8/P/fcc08+8IEPZKeddqrFvt6VmpubU19fn6ampnTp0qX0dt5z6upqf863/tv/7mamtWWetWemtWemtWWetWemtbU+5pmYaa1tyPNMzLTWzLP2zLT2zLS2zLP2zLS2/E1ae35H313WtBus1Z01v/vd79KvX780Nze3OL7VVltlv/32y1FHHZX/+7//W7cdAwAAAAAAbIDWKtZceOGFOf7441dbf+rr63PCCSdk0qRJNdscAAAAAADAe91axZp77703Bx544Os+P3jw4MyaNWudNnL22Wenrq4uY8eOrR6rVCoZP358evXqlQ4dOmTQoEG5//77W7xu2bJlOemkk9K9e/d06tQphxxySB577LEWaxobGzNy5MjU19envr4+I0eOzLPPPttizfz58zNs2LB06tQp3bt3z5gxY7J8+fJ1ei8AAAAAAABraq1izRNPPJE2bdq87vOtW7fOk08+udabuPvuu/Pf//3fq3zfzXnnnZdJkyZl8uTJufvuu9PQ0JADDjggzz33XHXN2LFjc91112XatGmZOXNmnn/++QwdOjQrV66srhk+fHhmz56d6dOnZ/r06Zk9e3ZGjhxZfX7lypU5+OCDs2TJksycOTPTpk3Ltddem5NPPnmt3wsAAAAAAMDaWKtY8773vS/33Xff6z7/l7/8JZtvvvlabeD555/PiBEj8sMf/jBdu3atHq9UKrnwwgvzn//5nznssMPSv3//XHHFFXnhhRdy9dVXJ0mampoyZcqUXHDBBdl///2zyy675Mc//nHuu+++/Pa3v02SPPjgg5k+fXp+9KMfZcCAARkwYEB++MMf5le/+lX++te/JklmzJiRBx54ID/+8Y+zyy67ZP/9988FF1yQH/7wh6t8P8+/WrZsWZqbm1s8AAAAAAAA1sZaxZqDDjooX/va1/Liiy+u8tzSpUvz9a9/PUOHDl2rDXzxi1/MwQcfnP3337/F8blz52bRokUZPHhw9Vi7du0ycODA3HbbbUmSWbNmZcWKFS3W9OrVK/3796+uuf3221NfX5899tijumbPPfdMfX19izX9+/dPr169qmuGDBmSZcuWveHHup199tnVj1arr6/PlltuuVbvHQAAAAAAoPXaLD7jjDPy85//PNttt12+9KUvpW/fvqmrq8uDDz6Y733ve1m5cmX+8z//c43PN23atPzpT3/K3XffvcpzixYtSpL07NmzxfGePXtm3rx51TVt27ZtcUfOq2teff2iRYvSo0ePVc7fo0ePFmtee52uXbumbdu21TWrc/rpp+crX/lK9efm5mbBBgAAAAAAWCtrFWt69uyZ2267LV/4whdy+umnp1KpJEnq6uoyZMiQfP/7318leryeBQsW5N///d8zY8aMtG/f/nXX1dXVtfi5Uqmscuy1XrtmdevXZc1rtWvXLu3atXvDvQAAAAAAALyRtYo1SbL11lvn17/+dRobG/P3v/89lUolffr0WeXuljcza9asLF68OLvuumv12MqVK/OHP/whkydPrn6fzKJFi1p8D87ixYurQaihoSHLly9PY2Nji+svXrw4e+21V3XNE088scr1n3zyyRbnufPOO1s839jYmBUrVqxxfAIAAAAAAFgXa/WdNf+qa9eu2X333fORj3xkrUNNkuy333657777Mnv27Opjt912y4gRIzJ79uy8//3vT0NDQ2666abqa5YvX55bb721GmJ23XXXtGnTpsWahQsXZs6cOdU1AwYMSFNTU+66667qmjvvvDNNTU0t1syZMycLFy6srpkxY0batWvXIiYBAAAAAADU2lrfWVMrnTt3Tv/+/Vsc69SpUzbddNPq8bFjx2bChAnp06dP+vTpkwkTJqRjx44ZPnx4kqS+vj7HHntsTj755Gy66abp1q1bxo0blx133DH7779/kmT77bfPgQcemOOPPz6XXHJJkuTzn/98hg4dmr59+yZJBg8enH79+mXkyJGZOHFinnnmmYwbNy7HH398unTp8naNBAAAAAAA2AAVizVr4tRTT83SpUszevToNDY2Zo899siMGTPSuXPn6ppvf/vbad26dQ4//PAsXbo0++23Xy6//PK0atWqumbq1KkZM2ZMBg8enCQ55JBDMnny5OrzrVq1yg033JDRo0dn7733TocOHTJ8+PCcf/75b9+bBQAAAAAANkh1lUqlUnoT7xXNzc2pr69PU1OTO3LWg7q62p9zQ//tN9PaMs/aM9PaM9PaMs/aM9PaWh/zTMy01jbkeSZmWmvmWXtmWntmWlvmWXtmWlv+Jq09v6PvLmvaDdb5O2sAAAAAAAB468QaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgsQaAAAAAACAgorGmosvvjg77bRTunTpki5dumTAgAG58cYbq89XKpWMHz8+vXr1SocOHTJo0KDcf//9Lc6xbNmynHTSSenevXs6deqUQw45JI899liLNY2NjRk5cmTq6+tTX1+fkSNH5tlnn22xZv78+Rk2bFg6deqU7t27Z8yYMVm+fPl6e+8AAAAAAABJ4VizxRZb5Jxzzsk999yTe+65Jx//+Mdz6KGHVoPMeeedl0mTJmXy5Mm5++6709DQkAMOOCDPPfdc9Rxjx47Nddddl2nTpmXmzJl5/vnnM3To0KxcubK6Zvjw4Zk9e3amT5+e6dOnZ/b/1969R1dd3/n+f2W4RESIIELIFJAqogilFj0KesRWDToCVjvVDjMRqvVyvFIvtbanp07PFKpWbc+wtLWjYK2Vrlle2qmKYFVGj6KIZbyM2hsWOoI4FQMiBYT9+2NO82vAet3wYcPjsVbWMnt/svPO2w0ryZNvsnBhWlpa2u7fsGFDjj322KxevToPP/xwZs6cmdtuuy0XXnjh1lsGAAAAAACwQ6qrVCqV0kP8qZ49e+bKK6/MKaeckqampkyePDmXXHJJkv+6iqZPnz65/PLLc8YZZ6S1tTW77757br755px00klJkpdeein9+vXL3XffnTFjxuS5557LkCFDMm/evBx00EFJknnz5mXkyJF5/vnnM3jw4Nxzzz0ZO3ZslixZkqampiTJzJkzM2nSpCxfvjzdu3d/V7OvXLkyDQ0NaW1tfddvw7tXV1f9x9y2nv1bn51Wl31Wn51Wn51Wl31Wn51W15bYZ2Kn1bYj7zOx02qzz+qz0+qz0+qyz+qz0+ryOWn1eY7WlnfbDbaZ31mzYcOGzJw5M6tXr87IkSOzaNGiLFu2LM3NzW1n6uvrM3r06DzyyCNJkgULFmT9+vXtzjQ1NWXo0KFtZx599NE0NDS0hZokOfjgg9PQ0NDuzNChQ9tCTZKMGTMma9euzYIFC/7szGvXrs3KlSvbvQAAAAAAALwXxWPN008/nV122SX19fU588wzc8cdd2TIkCFZtmxZkqRPnz7tzvfp06ftvmXLlqVz587p0aPH257p3bv3Zu+3d+/e7c5s+n569OiRzp07t515K1OnTm37PTgNDQ3p16/fe/zoAQAAAACAHV3xWDN48OAsXLgw8+bNy//4H/8jEydOzL//+7+33V+3yTVdlUpls9s2temZtzr/fs5s6tJLL01ra2vby5IlS952LgAAAAAAgE0VjzWdO3fOXnvtlQMOOCBTp07N8OHD8+1vfzuNjY1JstmVLcuXL2+7CqaxsTHr1q3LihUr3vbMyy+/vNn7feWVV9qd2fT9rFixIuvXr9/sips/VV9fn+7du7d7AQAAAAAAeC+Kx5pNVSqVrF27NgMHDkxjY2PmzJnTdt+6desyd+7cjBo1KkkyYsSIdOrUqd2ZpUuX5plnnmk7M3LkyLS2tubxxx9vO/PYY4+ltbW13ZlnnnkmS5cubTsze/bs1NfXZ8SIEVv04wUAAAAAAHZsHUu+8y996Us55phj0q9fv6xatSozZ87Mgw8+mFmzZqWuri6TJ0/OlClTMmjQoAwaNChTpkzJzjvvnAkTJiRJGhoacuqpp+bCCy/Mbrvtlp49e+aiiy7KsGHDcuSRRyZJ9t133xx99NE57bTT8t3vfjdJcvrpp2fs2LEZPHhwkqS5uTlDhgxJS0tLrrzyyrz66qu56KKLctppp7laBgAAAAAA2KKKxpqXX345LS0tWbp0aRoaGvKRj3wks2bNylFHHZUk+cIXvpA1a9bkrLPOyooVK3LQQQdl9uzZ6datW9tjXHPNNenYsWNOPPHErFmzJkcccURmzJiRDh06tJ255ZZbct5556W5uTlJMn78+EybNq3t/g4dOuSuu+7KWWedlUMOOSRdunTJhAkT8s1vfnMrbQIAAAAAANhR1VUqlUrpIbYXK1euTENDQ1pbW12RswXU1VX/MXf0Z7+dVpd9Vp+dVp+dVpd9Vp+dVteW2Gdip9W2I+8zsdNqs8/qs9Pqs9Pqss/qs9Pq8jlp9XmO1pZ32w22ud9ZAwAAAAAAsCMRawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoqGmumTp2aAw88MN26dUvv3r3zyU9+Mi+88EK7M5VKJZdddlmamprSpUuXHH744Xn22WfbnVm7dm3OPffc9OrVK127ds348ePzu9/9rt2ZFStWpKWlJQ0NDWloaEhLS0tee+21dmcWL16ccePGpWvXrunVq1fOO++8rFu3bot87AAAAAAAAEnhWDN37tycffbZmTdvXubMmZM333wzzc3NWb16dduZK664IldffXWmTZuW+fPnp7GxMUcddVRWrVrVdmby5Mm54447MnPmzDz88MN5/fXXM3bs2GzYsKHtzIQJE7Jw4cLMmjUrs2bNysKFC9PS0tJ2/4YNG3Lsscdm9erVefjhhzNz5szcdtttufDCC7fOMgAAAAAAgB1SXaVSqZQe4o9eeeWV9O7dO3Pnzs1hhx2WSqWSpqamTJ48OZdcckmS/7qKpk+fPrn88stzxhlnpLW1NbvvvntuvvnmnHTSSUmSl156Kf369cvdd9+dMWPG5LnnnsuQIUMyb968HHTQQUmSefPmZeTIkXn++eczePDg3HPPPRk7dmyWLFmSpqamJMnMmTMzadKkLF++PN27d3/H+VeuXJmGhoa0tra+q/O8N3V11X/MbefZX4adVpd9Vp+dVp+dVpd9Vp+dVteW2Gdip9W2I+8zsdNqs8/qs9Pqs9Pqss/qs9Pq8jlp9XmO1pZ32w22qd9Z09ramiTp2bNnkmTRokVZtmxZmpub287U19dn9OjReeSRR5IkCxYsyPr169udaWpqytChQ9vOPProo2loaGgLNUly8MEHp6Ghod2ZoUOHtoWaJBkzZkzWrl2bBQsWvOW8a9euzcqVK9u9AAAAAAAAvBfbTKypVCq54IILcuihh2bo0KFJkmXLliVJ+vTp0+5snz592u5btmxZOnfunB49erztmd69e2/2Pnv37t3uzKbvp0ePHuncuXPbmU1NnTq17XfgNDQ0pF+/fu/1wwYAAAAAAHZw20ysOeecc/LUU0/l1ltv3ey+uk2u66pUKpvdtqlNz7zV+fdz5k9deumlaW1tbXtZsmTJ284EAAAAAACwqW0i1px77rn5yU9+kgceeCAf+tCH2m5vbGxMks2ubFm+fHnbVTCNjY1Zt25dVqxY8bZnXn755c3e7yuvvNLuzKbvZ8WKFVm/fv1mV9z8UX19fbp3797uBQAAAAAA4L0oGmsqlUrOOeec3H777bn//vszcODAdvcPHDgwjY2NmTNnTttt69aty9y5czNq1KgkyYgRI9KpU6d2Z5YuXZpnnnmm7czIkSPT2tqaxx9/vO3MY489ltbW1nZnnnnmmSxdurTtzOzZs1NfX58RI0ZU/4MHAAAAAABI0rHkOz/77LPzwx/+MD/+8Y/TrVu3titbGhoa0qVLl9TV1WXy5MmZMmVKBg0alEGDBmXKlCnZeeedM2HChLazp556ai688MLstttu6dmzZy666KIMGzYsRx55ZJJk3333zdFHH53TTjst3/3ud5Mkp59+esaOHZvBgwcnSZqbmzNkyJC0tLTkyiuvzKuvvpqLLroop512mitmAAAAAACALaZorLnuuuuSJIcffni726dPn55JkyYlSb7whS9kzZo1Oeuss7JixYocdNBBmT17drp169Z2/pprrknHjh1z4oknZs2aNTniiCMyY8aMdOjQoe3MLbfckvPOOy/Nzc1JkvHjx2fatGlt93fo0CF33XVXzjrrrBxyyCHp0qVLJkyYkG9+85tb6KMHAAAAAABI6iqVSqX0ENuLlStXpqGhIa2tra7G2QLq6qr/mDv6s99Oq8s+q89Oq89Oq8s+q89Oq2tL7DOx02rbkfeZ2Gm12Wf12Wn12Wl12Wf12Wl1+Zy0+jxHa8u77QZFf2cNAAAAAADAjk6sAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKEisAQAAAAAAKKhj6QFga6qkrv0NdW997r09aKUKDwJQGzb7ezTxdykAAADAByTWAAAAbEX+AREAwHvjHw6yIxBrgA/ENxuqyz4BAADY2nwtCjsef+63PWINAADbFV90wI7Hn/vqss/qs9Pqs1O2dZ6jwHsl1gCwXfMJMts6z1EAAADgL0oPAAAAAAAAsCNzZQ0AAPBn+WWuAAAAW54rawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoSawAAAAAAAAoqGmv+9V//NePGjUtTU1Pq6upy5513tru/UqnksssuS1NTU7p06ZLDDz88zz77bLsza9euzbnnnptevXqla9euGT9+fH73u9+1O7NixYq0tLSkoaEhDQ0NaWlpyWuvvdbuzOLFizNu3Lh07do1vXr1ynnnnZd169ZtiQ8bAAAAAACgTdFYs3r16gwfPjzTpk17y/uvuOKKXH311Zk2bVrmz5+fxsbGHHXUUVm1alXbmcmTJ+eOO+7IzJkz8/DDD+f111/P2LFjs2HDhrYzEyZMyMKFCzNr1qzMmjUrCxcuTEtLS9v9GzZsyLHHHpvVq1fn4YcfzsyZM3Pbbbflwgsv3HIfPAAAAAAAQJK6SqVSKT1EktTV1eWOO+7IJz/5yST/dVVNU1NTJk+enEsuuSTJf11F06dPn1x++eU544wz0tramt133z0333xzTjrppCTJSy+9lH79+uXuu+/OmDFj8txzz2XIkCGZN29eDjrooCTJvHnzMnLkyDz//PMZPHhw7rnnnowdOzZLlixJU1NTkmTmzJmZNGlSli9fnu7du7+rj2HlypVpaGhIa2vru34b3r26ug/+GJVU4UE2e9Bt4o/Q+2Kn1WWf1Wen1fdBd7pF9pnU7E49R6vPTqtrm91nYqfVVqP7TOy02uyz+uy0+uy0uuyz+uy0urbZfSZ2Wm01us8t7d12g232d9YsWrQoy5YtS3Nzc9tt9fX1GT16dB555JEkyYIFC7J+/fp2Z5qamjJ06NC2M48++mgaGhraQk2SHHzwwWloaGh3ZujQoW2hJknGjBmTtWvXZsGCBX92xrVr12blypXtXgAAAAAAAN6LbTbWLFu2LEnSp0+fdrf36dOn7b5ly5alc+fO6dGjx9ue6d2792aP37t373ZnNn0/PXr0SOfOndvOvJWpU6e2/R6choaG9OvX7z1+lAAAAAAAwI5um401f1S3yTVdlUpls9s2temZtzr/fs5s6tJLL01ra2vby5IlS952LgAAAAAAgE1ts7GmsbExSTa7smX58uVtV8E0NjZm3bp1WbFixdueefnllzd7/FdeeaXdmU3fz4oVK7J+/frNrrj5U/X19enevXu7FwAAAAAAgPdim401AwcOTGNjY+bMmdN227p16zJ37tyMGjUqSTJixIh06tSp3ZmlS5fmmWeeaTszcuTItLa25vHHH28789hjj6W1tbXdmWeeeSZLly5tOzN79uzU19dnxIgRW/TjBAAAAAAAdmwdS77z119/Pb/61a/aXl+0aFEWLlyYnj17pn///pk8eXKmTJmSQYMGZdCgQZkyZUp23nnnTJgwIUnS0NCQU089NRdeeGF222239OzZMxdddFGGDRuWI488Mkmy77775uijj85pp52W7373u0mS008/PWPHjs3gwYOTJM3NzRkyZEhaWlpy5ZVX5tVXX81FF12U0047zdUyAAAAAADAFlU01jzxxBP5+Mc/3vb6BRdckCSZOHFiZsyYkS984QtZs2ZNzjrrrKxYsSIHHXRQZs+enW7durW9zTXXXJOOHTvmxBNPzJo1a3LEEUdkxowZ6dChQ9uZW265Jeedd16am5uTJOPHj8+0adPa7u/QoUPuuuuunHXWWTnkkEPSpUuXTJgwId/85je39AoAAAAAAIAdXF2lUqmUHmJ7sXLlyjQ0NKS1tdUVOVtAXd0Hf4xKqvAgmz1o7f4RstPqss/qs9Pq+6A73SL7TGp2p56j1Wen1bXN7jOx02qr0X0mdlpt9ll9dlp9dlpd9ll9dlpd2+w+Ezutthrd55b2brvBNvs7awAAAAAAAHYEYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYg0AAAAAAEBBYs0mrr322gwcODA77bRTRowYkYceeqj0SAAAAAAAwHZMrPkTP/rRjzJ58uR8+ctfzs9//vP89//+33PMMcdk8eLFpUcDAAAAAAC2U2LNn7j66qtz6qmn5nOf+1z23XfffOtb30q/fv1y3XXXlR4NAAAAAADYTnUsPcC2Yt26dVmwYEG++MUvtru9ubk5jzzyyFu+zdq1a7N27dq211tbW5MkK1eu3HKD1qiGqQ1VeJTWD/wIW+T/TKH/33ZaXfZZfXZafdvCTrfYR+7PfXXtwM/RxE7b20b3mfhzX201+xxN7PT/Z5/VZ6fVZ6fVZZ/VZ6fV53PS6vIc3fH8sRdUKpW3PSfW/D//+Z//mQ0bNqRPnz7tbu/Tp0+WLVv2lm8zderU/P3f//1mt/fr12+LzMgH/4usGn8Vbv6gW+RRtxI7rS77rD47rb4PNvsW+8hrdqeeo9Vnp9W1je4zsdNqq9l9JnZabfZZfXZafXZaXfZZfXZaXdvoPhM7rbaa3efWsWrVqjS8zY7Emk3U1dW1e71SqWx22x9deumlueCCC9pe37hxY1599dXstttuf/ZteG9WrlyZfv36ZcmSJenevXvpcbYLdlpd9ll9dlpd9ll9dlp9dlpd9ll9dlp9dlpd9ll9dlpd9ll9dlp9dlpd9ll9dlq7KpVKVq1alaamprc9J9b8P7169UqHDh02u4pm+fLlm11t80f19fWpr69vd9uuu+66pUbcoXXv3t1fQlVmp9Vln9Vnp9Vln9Vnp9Vnp9Vln9Vnp9Vnp9Vln9Vnp9Vln9Vnp9Vnp9Vln9Vnp7Xp7a6o+aO/2Apz1ITOnTtnxIgRmTNnTrvb58yZk1GjRhWaCgAAAAAA2N65suZPXHDBBWlpackBBxyQkSNH5vrrr8/ixYtz5plnlh4NAAAAAADYTok1f+Kkk07K73//+3zta1/L0qVLM3To0Nx9990ZMGBA6dF2WPX19fnqV7+62Y+b4/2z0+qyz+qz0+qyz+qz0+qz0+qyz+qz0+qz0+qyz+qz0+qyz+qz0+qz0+qyz+qz0+1fXaVSqZQeAgAAAAAAYEfld9YAAAAAAAAUJNYAAAAAAAAUJNYAAAAAAAAUJNYAAAAAAAAUJNawTfrXf/3XjBs3Lk1NTamrq8udd95ZeqSaNnXq1Bx44IHp1q1bevfunU9+8pN54YUXSo9V06677rp85CMfSffu3dO9e/eMHDky99xzT+mxthtTp05NXV1dJk+eXHqUmnXZZZelrq6u3UtjY2PpsWref/zHf+Tv/u7vsttuu2XnnXfORz/60SxYsKD0WDVpjz322Ow5WldXl7PPPrv0aDXrzTffzP/8n/8zAwcOTJcuXfLhD384X/va17Jx48bSo9WsVatWZfLkyRkwYEC6dOmSUaNGZf78+aXHqhnv9Dl9pVLJZZddlqampnTp0iWHH354nn322TLD1oh32untt9+eMWPGpFevXqmrq8vChQuLzFlL3m6n69evzyWXXJJhw4ala9euaWpqysknn5yXXnqp3MDbuHd6jl522WXZZ5990rVr1/To0SNHHnlkHnvssTLD1oj38v2RM844I3V1dfnWt7611earNe+0z0mTJm32+enBBx9cZtga8W6eo88991zGjx+fhoaGdOvWLQcffHAWL1689YetEe+007f6Oqquri5XXnllmYGpGrGGbdLq1aszfPjwTJs2rfQo24W5c+fm7LPPzrx58zJnzpy8+eabaW5uzurVq0uPVrM+9KEP5Rvf+EaeeOKJPPHEE/nEJz6R4447zjcYqmD+/Pm5/vrr85GPfKT0KDVvv/32y9KlS9tenn766dIj1bQVK1bkkEMOSadOnXLPPffk3//933PVVVdl1113LT1aTZo/f3675+ecOXOSJJ/+9KcLT1a7Lr/88nznO9/JtGnT8txzz+WKK67IlVdemX/8x38sPVrN+tznPpc5c+bk5ptvztNPP53m5uYceeSR+Y//+I/So9WEd/qc/oorrsjVV1+dadOmZf78+WlsbMxRRx2VVatWbeVJa8c77XT16tU55JBD8o1vfGMrT1a73m6nb7zxRp588sl85StfyZNPPpnbb789v/jFLzJ+/PgCk9aGd3qO7r333pk2bVqefvrpPPzww9ljjz3S3NycV155ZStPWjve7fdH7rzzzjz22GNpamraSpPVpnezz6OPPrrd56l33333Vpyw9rzTTn/961/n0EMPzT777JMHH3ww//Zv/5avfOUr2WmnnbbypLXjnXb6p8/PpUuX5sYbb0xdXV0+9alPbeVJqboKbOOSVO64447SY2xXli9fXklSmTt3bulRtis9evSo/NM//VPpMWraqlWrKoMGDarMmTOnMnr06Mr5559feqSa9dWvfrUyfPjw0mNsVy655JLKoYceWnqM7db5559f2XPPPSsbN24sPUrNOvbYYyunnHJKu9tOOOGEyt/93d8Vmqi2vfHGG5UOHTpUfvrTn7a7ffjw4ZUvf/nLhaaqXZt+Tr9x48ZKY2Nj5Rvf+EbbbX/4wx8qDQ0Nle985zsFJqw9b/d10qJFiypJKj//+c+36ky17t187fn4449XklR++9vfbp2hati72Wdra2slSeW+++7bOkPVuD+309/97neVv/zLv6w888wzlQEDBlSuueaarT5bLXqrfU6cOLFy3HHHFZlne/BWOz3ppJN8PvoBvJu/S4877rjKJz7xia0zEFuUK2tgB9Ta2pok6dmzZ+FJtg8bNmzIzJkzs3r16owcObL0ODXt7LPPzrHHHpsjjzyy9CjbhV/+8pdpamrKwIED85nPfCa/+c1vSo9U037yk5/kgAMOyKc//en07t07+++/f773ve+VHmu7sG7duvzgBz/IKaeckrq6utLj1KxDDz00P/vZz/KLX/wiSfJv//Zvefjhh/NXf/VXhSerTW+++WY2bNiw2b/67NKlSx5++OFCU20/Fi1alGXLlqW5ubnttvr6+owePTqPPPJIwcng7bW2tqaurs6VtVWwbt26XH/99WloaMjw4cNLj1OzNm7cmJaWllx88cXZb7/9So+zXXjwwQfTu3fv7L333jnttNOyfPny0iPVrI0bN+auu+7K3nvvnTFjxqR379456KCD/LqDKnr55Zdz11135dRTTy09ClUg1sAOplKp5IILLsihhx6aoUOHlh6npj399NPZZZddUl9fnzPPPDN33HFHhgwZUnqsmjVz5sw8+eSTmTp1aulRtgsHHXRQvv/97+fee+/N9773vSxbtiyjRo3K73//+9Kj1azf/OY3ue666zJo0KDce++9OfPMM3Peeefl+9//funRat6dd96Z1157LZMmTSo9Sk275JJL8jd/8zfZZ5990qlTp+y///6ZPHly/uZv/qb0aDWpW7duGTlyZP73//7feemll7Jhw4b84Ac/yGOPPZalS5eWHq/mLVu2LEnSp0+fdrf36dOn7T7Y1vzhD3/IF7/4xUyYMCHdu3cvPU7N+ulPf5pddtklO+20U6655prMmTMnvXr1Kj1Wzbr88svTsWPHnHfeeaVH2S4cc8wxueWWW3L//ffnqquuyvz58/OJT3wia9euLT1aTVq+fHlef/31fOMb38jRRx+d2bNn5/jjj88JJ5yQuXPnlh5vu3DTTTelW7duOeGEE0qPQhV0LD0AsHWdc845eeqpp/yL0CoYPHhwFi5cmNdeey233XZbJk6cmLlz5wo278OSJUty/vnnZ/bs2X5ubZUcc8wxbf89bNiwjBw5MnvuuWduuummXHDBBQUnq10bN27MAQcckClTpiRJ9t9//zz77LO57rrrcvLJJxeerrbdcMMNOeaYY/yM9Q/oRz/6UX7wgx/khz/8Yfbbb78sXLgwkydPTlNTUyZOnFh6vJp0880355RTTslf/uVfpkOHDvnYxz6WCRMm5Mknnyw92nZj06vpKpWKK+zYJq1fvz6f+cxnsnHjxlx77bWlx6lpH//4x7Nw4cL853/+Z773ve/lxBNPzGOPPZbevXuXHq3mLFiwIN/+9rfz5JNP+ruzSk466aS2/x46dGgOOOCADBgwIHfddZdvhr8PGzduTJIcd9xx+fznP58k+ehHP5pHHnkk3/nOdzJ69OiS420Xbrzxxvzt3/6t76VsJ1xZAzuQc889Nz/5yU/ywAMP5EMf+lDpcWpe586ds9dee+WAAw7I1KlTM3z48Hz7298uPVZNWrBgQZYvX54RI0akY8eO6dixY+bOnZv/83/+Tzp27JgNGzaUHrHmde3aNcOGDcsvf/nL0qPUrL59+24WY/fdd98sXry40ETbh9/+9re577778rnPfa70KDXv4osvzhe/+MV85jOfybBhw9LS0pLPf/7zrlj8APbcc8/MnTs3r7/+epYsWZLHH38869evz8CBA0uPVvMaGxuTZLOraJYvX77Z1TZQ2vr163PiiSdm0aJFmTNnjqtqPqCuXbtmr732ysEHH5wbbrghHTt2zA033FB6rJr00EMPZfny5enfv3/b11G//e1vc+GFF2aPPfYoPd52oW/fvhkwYICvo96nXr16pWPHjr6O2kIeeuihvPDCC76W2o6INbADqFQqOeecc3L77bfn/vvv9w2GLaRSqbg0+n064ogj8vTTT2fhwoVtLwcccED+9m//NgsXLkyHDh1Kj1jz1q5dm+eeey59+/YtPUrNOuSQQ/LCCy+0u+0Xv/hFBgwYUGii7cP06dPTu3fvHHvssaVHqXlvvPFG/uIv2n9636FDh7Z/0cj717Vr1/Tt2zcrVqzIvffem+OOO670SDVv4MCBaWxszJw5c9puW7duXebOnZtRo0YVnAza+2Oo+eUvf5n77rsvu+22W+mRtju+jnr/Wlpa8tRTT7X7OqqpqSkXX3xx7r333tLjbRd+//vfZ8mSJb6Oep86d+6cAw880NdRW8gNN9yQESNG+L1f2xE/Bo1t0uuvv55f/epXba8vWrQoCxcuTM+ePdO/f/+Ck9Wms88+Oz/84Q/z4x//ON26dWv7F4wNDQ3p0qVL4elq05e+9KUcc8wx6devX1atWpWZM2fmwQcfzKxZs0qPVpO6deu22e9Q6tq1a3bbbTe/W+l9uuiiizJu3Lj0798/y5cvzz/8wz9k5cqVfhTSB/D5z38+o0aNypQpU3LiiSfm8ccfz/XXX5/rr7++9Gg1a+PGjZk+fXomTpyYjh19WvpBjRs3Ll//+tfTv3//7Lfffvn5z3+eq6++Oqecckrp0WrWvffem0qlksGDB+dXv/pVLr744gwePDif/exnS49WE97pc/rJkydnypQpGTRoUAYNGpQpU6Zk5513zoQJEwpOvW17p52++uqrWbx4cV566aUkafvmWGNjY9vVTLT3djttamrKX//1X+fJJ5/MT3/602zYsKHta6mePXumc+fOpcbeZr3dPnfbbbd8/etfz/jx49O3b9/8/ve/z7XXXpvf/e53+fSnP11w6m3bO/253zQgdurUKY2NjRk8ePDWHrUmvN0+e/bsmcsuuyyf+tSn0rdv37z44ov50pe+lF69euX4448vOPW27Z2eoxdffHFOOumkHHbYYfn4xz+eWbNm5V/+5V/y4IMPlht6G/duvi+6cuXK/PM//3OuuuqqUmOyJVRgG/TAAw9Ukmz2MnHixNKj1aS32mWSyvTp00uPVrNOOeWUyoABAyqdO3eu7L777pUjjjiiMnv27NJjbVdGjx5dOf/880uPUbNOOumkSt++fSudOnWqNDU1VU444YTKs88+W3qsmvcv//IvlaFDh1bq6+sr++yzT+X6668vPVJNu/feeytJKi+88ELpUbYLK1eurJx//vmV/v37V3baaafKhz/84cqXv/zlytq1a0uPVrN+9KMfVT784Q9XOnfuXGlsbKycffbZlddee630WDXjnT6n37hxY+WrX/1qpbGxsVJfX1857LDDKk8//XTZobdx77TT6dOnv+X9X/3qV4vOvS17u50uWrToz34t9cADD5QefZv0dvtcs2ZN5fjjj680NTVVOnfuXOnbt29l/Pjxlccff7z02Nu09/r9kQEDBlSuueaarTpjLXm7fb7xxhuV5ubmyu67717p1KlTpX///pWJEydWFi9eXHrsbdq7eY7ecMMNlb322quy0047VYYPH1658847yw1cA97NTr/73e9WunTp4nPT7UxdpVKpVKH5AAAAAAAA8D74nTUAAAAAAAAFiTUAAAAAAAAFiTUAAAAAAAAFiTUAAAAAAAAFiTUAAAAAAAAFiTUAAAAAAAAFiTUAAAAAAAAFiTUAAAAAAAAFiTUAAAA15PDDD8/kyZNLjwEAAFSRWAMAANS8SZMmpa6uLmeeeeZm95111lmpq6vLpEmTkvz52HHnnXemrq4uSXLVVVeloaEhb7zxxmbn/vCHP2TXXXfN1Vdf/Y5z1dXV5c4773xPHwsAALDjEWsAAIDtQr9+/TJz5sysWbOm7bY//OEPufXWW9O/f//39Fgnn3xy1qxZk9tuu22z+2677ba88cYbaWlp+cAzAwAAJGINAACwnfjYxz6W/v375/bbb2+77fbbb0+/fv2y//77v6fH2n333TNu3LjceOONm9134403Zvz48dl9993f9jH22GOPJMnxxx+furq6ttd//etf57jjjkufPn2yyy675MADD8x9993X7m2vvfbaDBo0KDvttFP69OmTv/7rv/6z72fWrFlpaGjI97///STJgw8+mP/23/5bunbtml133TWHHHJIfvvb376Hjx4AANjaxBoAAGC78dnPfjbTp09ve/3GG2/MKaec8r4e69RTT83cuXOzaNGitttefPHFPPDAAzn11FPf8e3nz5+fJJk+fXqWLl3a9vrrr7+ev/qrv8p9992Xn//85xkzZkzGjRuXxYsXJ0meeOKJnHfeefna176WF154IbNmzcphhx32lu9j5syZOfHEE/P9738/J598ct5888188pOfzOjRo/PUU0/l0Ucfzemnn972490AAIBtU8fSAwAAAFRLS0tLLr300rz44oupq6vL//2//zczZ87Mgw8++J4fa8yYMWlqasqMGTPy93//90n+K7w0NTWlubn5Hd/+j1fe7LrrrmlsbGy7ffjw4Rk+fHjb6//wD/+QO+64Iz/5yU9yzjnnZPHixenatWvGjh2bbt26ZcCAAW95ZdC1116bL33pS/nxj3+cj3/840mSlStXprW1NWPHjs2ee+6ZJNl3333f88cOAABsXa6sAQAAthu9evXKsccem5tuuinTp0/Psccem169er2vx+rQoUMmTpyYGTNmZOPGjalUKrnpppsyadKkdOjQ4X3PuHr16nzhC1/IkCFDsuuuu2aXXXbJ888/33ZlzVFHHZUBAwbkwx/+cFpaWnLLLbfkjTfeaPcYt912WyZPnpzZs2e3hZok6dmzZyZNmtR2tc63v/3tLF269H3PCgAAbB1iDQAAsF055ZRTMmPGjNx0001v+SPQunfvntbW1s1uf+2119K9e/fNHmvJkiW5//7787Of/SyLFy/OZz/72Q8038UXX5zbbrstX//61/PQQw9l4cKFGTZsWNatW5ck6datW5588snceuut6du3b/7X//pfGT58eF577bW2x/joRz+a3XffPdOnT0+lUmn3+NOnT8+jjz6aUaNG5Uc/+lH23nvvzJs37wPNDAAAbFliDQAAsF05+uijs27duqxbty5jxozZ7P599tknTzzxxGa3z58/P4MHD25325577pnRo0dn+vTpufHGG3P44Ye3/Xixd6NTp07ZsGFDu9seeuihTJo0Kccff3yGDRuWxsbGvPjii+3OdOzYMUceeWSuuOKKPPXUU3nxxRdz//33t5vrgQceyI9//OOce+65m73f/fffP5deemkeeeSRDB06ND/84Q/f9cwAAMDW53fWAAAA25UOHTrkueeea/vvTZ111lmZNm1azj777Jx++unp0qVL5syZkxtuuCE333zzZudPPfXUnHbaaUmSf/qnf3pPs+yxxx752c9+lkMOOST19fXp0aNH9tprr9x+++0ZN25c6urq8pWvfCUbN25se5uf/vSn+c1vfpPDDjssPXr0yN13352NGzduFpL23nvvPPDAAzn88MPTsWPHfOtb38qiRYty/fXXZ/z48WlqasoLL7yQX/ziFzn55JPf09wAAMDW5coaAABgu9O9e/fNfqTZH+2xxx556KGH8utf/zrNzc058MADM2PGjMyYMSOf/vSnNzv/qU99KvX19amvr88JJ5zwnua46qqrMmfOnPTr1y/7779/kuSaa65Jjx49MmrUqIwbNy5jxozJxz72sba32XXXXXP77bfnE5/4RPbdd9985zvfya233pr99ttvs8cfPHhw7r///tx666258MILs/POO+f555/Ppz71qey99945/fTTc8455+SMM854T3MDAABbV11l0x9wDAAAAAAAwFbjyhoAAAAAAICCxBoAAID34ZZbbskuu+zyli9v9SPLAAAA/hw/Bg0AAOB9WLVqVV5++eW3vK9Tp04ZMGDAVp4IAACoVWINAAAAAABAQX4MGgAAAAAAQEFiDQAAAAAAQEFiDQAAAAAAQEFiDQAAAAAAQEFiDQAAAAAAQEFiDQAAAAAAQEFiDQAAAAAAQEH/H9cYC9zD4zd2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 2000x1500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Importing the matplotlib library\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "# Declaring the figure or the plot (y, x) or (width, height)\n",
    "plt.figure(figsize=[20, 15])\n",
    "\n",
    "X = np.arange(1,len(muv_tasks)+1)\n",
    "plt.bar(X + 0.2, one, color = 'g', width = 0.25)\n",
    "plt.bar(X + 0.4, zero, color = 'b', width = 0.25)\n",
    "plt.bar(X + 0.6, nan, color = 'r', width = 0.25)\n",
    "\n",
    "# Creating the legend of the bars in the plot\n",
    "plt.legend(['Active' , 'Inactive' ,'NAN'])\n",
    "# Overiding the x axis with the country names\n",
    "plt.xticks([i + 0.25 for i in range(1,18)], X)\n",
    "# Giving the tilte for the plot\n",
    "plt.title(\"MUV dataset diagram\")\n",
    "# Namimg the x and y axis\n",
    "plt.xlabel('MUV_tasks')\n",
    "plt.ylabel('Cases')\n",
    "# Saving the plot as a 'png'\n",
    "plt.savefig('4BarPlot.png')\n",
    "# Displaying the bar plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "6grIE_JeqkUZ"
   },
   "source": [
    "# Required functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "hidden": true,
    "id": "IzllOg474i99"
   },
   "outputs": [],
   "source": [
    "from dgllife.model import MLPPredictor\n",
    "\n",
    "def create_dataset_with_gcn(dataset, class_embed_vector, GCN, tasks, numberTask):\n",
    "\n",
    "    created_data = []\n",
    "    data = np.arange(len(tasks))\n",
    "    onehot_encoded = to_categorical(data)\n",
    "\n",
    "    for i, data in enumerate(dataset):\n",
    "        smiles, g, label, mask = data\n",
    "        g = g.to(device)\n",
    "        g = dgl.add_self_loop(g)\n",
    "        graph_feats = g.ndata.pop('h')\n",
    "        embbed = GCN(g, graph_feats)\n",
    "        embbed = embbed.to('cpu')\n",
    "        embbed = embbed.detach().numpy()\n",
    "        a = ( embbed, onehot_encoded[numberTask], class_embed_vector[numberTask], label, tasks[numberTask])\n",
    "        created_data.append(a)\n",
    "    print('Data created!!')\n",
    "    return created_data "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Calculation of embedded vectors for each class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MUV-466=> positive: 27 - negative: 14814\n",
      "MUV-548=> positive: 29 - negative: 14705\n",
      "MUV-600=> positive: 30 - negative: 14698\n",
      "MUV-644=> positive: 30 - negative: 14593\n",
      "MUV-652=> positive: 29 - negative: 14873\n",
      "MUV-689=> positive: 29 - negative: 14572\n",
      "MUV-692=> positive: 30 - negative: 14614\n",
      "MUV-712=> positive: 28 - negative: 14383\n",
      "MUV-713=> positive: 29 - negative: 14807\n",
      "MUV-733=> positive: 28 - negative: 14654\n",
      "MUV-737=> positive: 29 - negative: 14662\n",
      "MUV-810=> positive: 29 - negative: 14615\n",
      "MUV-832=> positive: 30 - negative: 14637\n",
      "MUV-846=> positive: 30 - negative: 14681\n",
      "MUV-852=> positive: 29 - negative: 14622\n",
      "MUV-858=> positive: 29 - negative: 14745\n",
      "MUV-859=> positive: 24 - negative: 14722\n"
     ]
    }
   ],
   "source": [
    "df_positive, df_negative = separate_active_and_inactive_data(df, muv_tasks)\n",
    "\n",
    "for i,d in enumerate(zip(df_positive,df_negative)):\n",
    "    print(f'{muv_tasks[i]}=> positive: {len(d[0])} - negative: {len(d[1])}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14814\n",
      "Processing molecule 2000/14814\n",
      "Processing molecule 3000/14814\n",
      "Processing molecule 4000/14814\n",
      "Processing molecule 5000/14814\n",
      "Processing molecule 6000/14814\n",
      "Processing molecule 7000/14814\n",
      "Processing molecule 8000/14814\n",
      "Processing molecule 9000/14814\n",
      "Processing molecule 10000/14814\n",
      "Processing molecule 11000/14814\n",
      "Processing molecule 12000/14814\n",
      "Processing molecule 13000/14814\n",
      "Processing molecule 14000/14814\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14705\n",
      "Processing molecule 2000/14705\n",
      "Processing molecule 3000/14705\n",
      "Processing molecule 4000/14705\n",
      "Processing molecule 5000/14705\n",
      "Processing molecule 6000/14705\n",
      "Processing molecule 7000/14705\n",
      "Processing molecule 8000/14705\n",
      "Processing molecule 9000/14705\n",
      "Processing molecule 10000/14705\n",
      "Processing molecule 11000/14705\n",
      "Processing molecule 12000/14705\n",
      "Processing molecule 13000/14705\n",
      "Processing molecule 14000/14705\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14698\n",
      "Processing molecule 2000/14698\n",
      "Processing molecule 3000/14698\n",
      "Processing molecule 4000/14698\n",
      "Processing molecule 5000/14698\n",
      "Processing molecule 6000/14698\n",
      "Processing molecule 7000/14698\n",
      "Processing molecule 8000/14698\n",
      "Processing molecule 9000/14698\n",
      "Processing molecule 10000/14698\n",
      "Processing molecule 11000/14698\n",
      "Processing molecule 12000/14698\n",
      "Processing molecule 13000/14698\n",
      "Processing molecule 14000/14698\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14593\n",
      "Processing molecule 2000/14593\n",
      "Processing molecule 3000/14593\n",
      "Processing molecule 4000/14593\n",
      "Processing molecule 5000/14593\n",
      "Processing molecule 6000/14593\n",
      "Processing molecule 7000/14593\n",
      "Processing molecule 8000/14593\n",
      "Processing molecule 9000/14593\n",
      "Processing molecule 10000/14593\n",
      "Processing molecule 11000/14593\n",
      "Processing molecule 12000/14593\n",
      "Processing molecule 13000/14593\n",
      "Processing molecule 14000/14593\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14873\n",
      "Processing molecule 2000/14873\n",
      "Processing molecule 3000/14873\n",
      "Processing molecule 4000/14873\n",
      "Processing molecule 5000/14873\n",
      "Processing molecule 6000/14873\n",
      "Processing molecule 7000/14873\n",
      "Processing molecule 8000/14873\n",
      "Processing molecule 9000/14873\n",
      "Processing molecule 10000/14873\n",
      "Processing molecule 11000/14873\n",
      "Processing molecule 12000/14873\n",
      "Processing molecule 13000/14873\n",
      "Processing molecule 14000/14873\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14572\n",
      "Processing molecule 2000/14572\n",
      "Processing molecule 3000/14572\n",
      "Processing molecule 4000/14572\n",
      "Processing molecule 5000/14572\n",
      "Processing molecule 6000/14572\n",
      "Processing molecule 7000/14572\n",
      "Processing molecule 8000/14572\n",
      "Processing molecule 9000/14572\n",
      "Processing molecule 10000/14572\n",
      "Processing molecule 11000/14572\n",
      "Processing molecule 12000/14572\n",
      "Processing molecule 13000/14572\n",
      "Processing molecule 14000/14572\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14614\n",
      "Processing molecule 2000/14614\n",
      "Processing molecule 3000/14614\n",
      "Processing molecule 4000/14614\n",
      "Processing molecule 5000/14614\n",
      "Processing molecule 6000/14614\n",
      "Processing molecule 7000/14614\n",
      "Processing molecule 8000/14614\n",
      "Processing molecule 9000/14614\n",
      "Processing molecule 10000/14614\n",
      "Processing molecule 11000/14614\n",
      "Processing molecule 12000/14614\n",
      "Processing molecule 13000/14614\n",
      "Processing molecule 14000/14614\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14383\n",
      "Processing molecule 2000/14383\n",
      "Processing molecule 3000/14383\n",
      "Processing molecule 4000/14383\n",
      "Processing molecule 5000/14383\n",
      "Processing molecule 6000/14383\n",
      "Processing molecule 7000/14383\n",
      "Processing molecule 8000/14383\n",
      "Processing molecule 9000/14383\n",
      "Processing molecule 10000/14383\n",
      "Processing molecule 11000/14383\n",
      "Processing molecule 12000/14383\n",
      "Processing molecule 13000/14383\n",
      "Processing molecule 14000/14383\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14807\n",
      "Processing molecule 2000/14807\n",
      "Processing molecule 3000/14807\n",
      "Processing molecule 4000/14807\n",
      "Processing molecule 5000/14807\n",
      "Processing molecule 6000/14807\n",
      "Processing molecule 7000/14807\n",
      "Processing molecule 8000/14807\n",
      "Processing molecule 9000/14807\n",
      "Processing molecule 10000/14807\n",
      "Processing molecule 11000/14807\n",
      "Processing molecule 12000/14807\n",
      "Processing molecule 13000/14807\n",
      "Processing molecule 14000/14807\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14654\n",
      "Processing molecule 2000/14654\n",
      "Processing molecule 3000/14654\n",
      "Processing molecule 4000/14654\n",
      "Processing molecule 5000/14654\n",
      "Processing molecule 6000/14654\n",
      "Processing molecule 7000/14654\n",
      "Processing molecule 8000/14654\n",
      "Processing molecule 9000/14654\n",
      "Processing molecule 10000/14654\n",
      "Processing molecule 11000/14654\n",
      "Processing molecule 12000/14654\n",
      "Processing molecule 13000/14654\n",
      "Processing molecule 14000/14654\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14662\n",
      "Processing molecule 2000/14662\n",
      "Processing molecule 3000/14662\n",
      "Processing molecule 4000/14662\n",
      "Processing molecule 5000/14662\n",
      "Processing molecule 6000/14662\n",
      "Processing molecule 7000/14662\n",
      "Processing molecule 8000/14662\n",
      "Processing molecule 9000/14662\n",
      "Processing molecule 10000/14662\n",
      "Processing molecule 11000/14662\n",
      "Processing molecule 12000/14662\n",
      "Processing molecule 13000/14662\n",
      "Processing molecule 14000/14662\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14615\n",
      "Processing molecule 2000/14615\n",
      "Processing molecule 3000/14615\n",
      "Processing molecule 4000/14615\n",
      "Processing molecule 5000/14615\n",
      "Processing molecule 6000/14615\n",
      "Processing molecule 7000/14615\n",
      "Processing molecule 8000/14615\n",
      "Processing molecule 9000/14615\n",
      "Processing molecule 10000/14615\n",
      "Processing molecule 11000/14615\n",
      "Processing molecule 12000/14615\n",
      "Processing molecule 13000/14615\n",
      "Processing molecule 14000/14615\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14637\n",
      "Processing molecule 2000/14637\n",
      "Processing molecule 3000/14637\n",
      "Processing molecule 4000/14637\n",
      "Processing molecule 5000/14637\n",
      "Processing molecule 6000/14637\n",
      "Processing molecule 7000/14637\n",
      "Processing molecule 8000/14637\n",
      "Processing molecule 9000/14637\n",
      "Processing molecule 10000/14637\n",
      "Processing molecule 11000/14637\n",
      "Processing molecule 12000/14637\n",
      "Processing molecule 13000/14637\n",
      "Processing molecule 14000/14637\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14681\n",
      "Processing molecule 2000/14681\n",
      "Processing molecule 3000/14681\n",
      "Processing molecule 4000/14681\n",
      "Processing molecule 5000/14681\n",
      "Processing molecule 6000/14681\n",
      "Processing molecule 7000/14681\n",
      "Processing molecule 8000/14681\n",
      "Processing molecule 9000/14681\n",
      "Processing molecule 10000/14681\n",
      "Processing molecule 11000/14681\n",
      "Processing molecule 12000/14681\n",
      "Processing molecule 13000/14681\n",
      "Processing molecule 14000/14681\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14622\n",
      "Processing molecule 2000/14622\n",
      "Processing molecule 3000/14622\n",
      "Processing molecule 4000/14622\n",
      "Processing molecule 5000/14622\n",
      "Processing molecule 6000/14622\n",
      "Processing molecule 7000/14622\n",
      "Processing molecule 8000/14622\n",
      "Processing molecule 9000/14622\n",
      "Processing molecule 10000/14622\n",
      "Processing molecule 11000/14622\n",
      "Processing molecule 12000/14622\n",
      "Processing molecule 13000/14622\n",
      "Processing molecule 14000/14622\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14745\n",
      "Processing molecule 2000/14745\n",
      "Processing molecule 3000/14745\n",
      "Processing molecule 4000/14745\n",
      "Processing molecule 5000/14745\n",
      "Processing molecule 6000/14745\n",
      "Processing molecule 7000/14745\n",
      "Processing molecule 8000/14745\n",
      "Processing molecule 9000/14745\n",
      "Processing molecule 10000/14745\n",
      "Processing molecule 11000/14745\n",
      "Processing molecule 12000/14745\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing molecule 13000/14745\n",
      "Processing molecule 14000/14745\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14722\n",
      "Processing molecule 2000/14722\n",
      "Processing molecule 3000/14722\n",
      "Processing molecule 4000/14722\n",
      "Processing molecule 5000/14722\n",
      "Processing molecule 6000/14722\n",
      "Processing molecule 7000/14722\n",
      "Processing molecule 8000/14722\n",
      "Processing molecule 9000/14722\n",
      "Processing molecule 10000/14722\n",
      "Processing molecule 11000/14722\n",
      "Processing molecule 12000/14722\n",
      "Processing molecule 13000/14722\n",
      "Processing molecule 14000/14722\n"
     ]
    }
   ],
   "source": [
    "dataset_positive = [DATASET(d,smiles_to_bigraph, AttentiveFPAtomFeaturizer(), cache_file_path = cache_path) for d in df_positive]\n",
    "dataset_negative = [DATASET(d,smiles_to_bigraph, AttentiveFPAtomFeaturizer(), cache_file_path = cache_path) for d in df_negative]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "class vector created!!\n"
     ]
    }
   ],
   "source": [
    "embed_class_muv = get_embedding_vector_class(dataset_positive, dataset_negative, radius=2, size = 512)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rINYL0dJAFdU"
   },
   "source": [
    "# **Association-based strategy with BioAct-Het**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zIKQi__XAcia"
   },
   "source": [
    "## Classification with BioAct-Het and AttentiveFp GCN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lDS5UguKr_x_",
    "outputId": "da58be7e-197e-4838-f5f1-a0b2d6b87cb2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading GCN_attentivefp_MUV_pre_trained.pth from https://data.dgl.ai/dgllife/pre_trained/gcn_attentivefp_muv.pth...\n",
      "Pretrained model loaded\n"
     ]
    }
   ],
   "source": [
    "model_name = 'GCN_attentivefp_MUV'\n",
    "gcn_model = get_muv_model(model_name)\n",
    "gcn_model.eval()\n",
    "gcn_model = gcn_model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14841\n",
      "Processing molecule 2000/14841\n",
      "Processing molecule 3000/14841\n",
      "Processing molecule 4000/14841\n",
      "Processing molecule 5000/14841\n",
      "Processing molecule 6000/14841\n",
      "Processing molecule 7000/14841\n",
      "Processing molecule 8000/14841\n",
      "Processing molecule 9000/14841\n",
      "Processing molecule 10000/14841\n",
      "Processing molecule 11000/14841\n",
      "Processing molecule 12000/14841\n",
      "Processing molecule 13000/14841\n",
      "Processing molecule 14000/14841\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14734\n",
      "Processing molecule 2000/14734\n",
      "Processing molecule 3000/14734\n",
      "Processing molecule 4000/14734\n",
      "Processing molecule 5000/14734\n",
      "Processing molecule 6000/14734\n",
      "Processing molecule 7000/14734\n",
      "Processing molecule 8000/14734\n",
      "Processing molecule 9000/14734\n",
      "Processing molecule 10000/14734\n",
      "Processing molecule 11000/14734\n",
      "Processing molecule 12000/14734\n",
      "Processing molecule 13000/14734\n",
      "Processing molecule 14000/14734\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14728\n",
      "Processing molecule 2000/14728\n",
      "Processing molecule 3000/14728\n",
      "Processing molecule 4000/14728\n",
      "Processing molecule 5000/14728\n",
      "Processing molecule 6000/14728\n",
      "Processing molecule 7000/14728\n",
      "Processing molecule 8000/14728\n",
      "Processing molecule 9000/14728\n",
      "Processing molecule 10000/14728\n",
      "Processing molecule 11000/14728\n",
      "Processing molecule 12000/14728\n",
      "Processing molecule 13000/14728\n",
      "Processing molecule 14000/14728\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14623\n",
      "Processing molecule 2000/14623\n",
      "Processing molecule 3000/14623\n",
      "Processing molecule 4000/14623\n",
      "Processing molecule 5000/14623\n",
      "Processing molecule 6000/14623\n",
      "Processing molecule 7000/14623\n",
      "Processing molecule 8000/14623\n",
      "Processing molecule 9000/14623\n",
      "Processing molecule 10000/14623\n",
      "Processing molecule 11000/14623\n",
      "Processing molecule 12000/14623\n",
      "Processing molecule 13000/14623\n",
      "Processing molecule 14000/14623\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14902\n",
      "Processing molecule 2000/14902\n",
      "Processing molecule 3000/14902\n",
      "Processing molecule 4000/14902\n",
      "Processing molecule 5000/14902\n",
      "Processing molecule 6000/14902\n",
      "Processing molecule 7000/14902\n",
      "Processing molecule 8000/14902\n",
      "Processing molecule 9000/14902\n",
      "Processing molecule 10000/14902\n",
      "Processing molecule 11000/14902\n",
      "Processing molecule 12000/14902\n",
      "Processing molecule 13000/14902\n",
      "Processing molecule 14000/14902\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14601\n",
      "Processing molecule 2000/14601\n",
      "Processing molecule 3000/14601\n",
      "Processing molecule 4000/14601\n",
      "Processing molecule 5000/14601\n",
      "Processing molecule 6000/14601\n",
      "Processing molecule 7000/14601\n",
      "Processing molecule 8000/14601\n",
      "Processing molecule 9000/14601\n",
      "Processing molecule 10000/14601\n",
      "Processing molecule 11000/14601\n",
      "Processing molecule 12000/14601\n",
      "Processing molecule 13000/14601\n",
      "Processing molecule 14000/14601\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14644\n",
      "Processing molecule 2000/14644\n",
      "Processing molecule 3000/14644\n",
      "Processing molecule 4000/14644\n",
      "Processing molecule 5000/14644\n",
      "Processing molecule 6000/14644\n",
      "Processing molecule 7000/14644\n",
      "Processing molecule 8000/14644\n",
      "Processing molecule 9000/14644\n",
      "Processing molecule 10000/14644\n",
      "Processing molecule 11000/14644\n",
      "Processing molecule 12000/14644\n",
      "Processing molecule 13000/14644\n",
      "Processing molecule 14000/14644\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14411\n",
      "Processing molecule 2000/14411\n",
      "Processing molecule 3000/14411\n",
      "Processing molecule 4000/14411\n",
      "Processing molecule 5000/14411\n",
      "Processing molecule 6000/14411\n",
      "Processing molecule 7000/14411\n",
      "Processing molecule 8000/14411\n",
      "Processing molecule 9000/14411\n",
      "Processing molecule 10000/14411\n",
      "Processing molecule 11000/14411\n",
      "Processing molecule 12000/14411\n",
      "Processing molecule 13000/14411\n",
      "Processing molecule 14000/14411\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14836\n",
      "Processing molecule 2000/14836\n",
      "Processing molecule 3000/14836\n",
      "Processing molecule 4000/14836\n",
      "Processing molecule 5000/14836\n",
      "Processing molecule 6000/14836\n",
      "Processing molecule 7000/14836\n",
      "Processing molecule 8000/14836\n",
      "Processing molecule 9000/14836\n",
      "Processing molecule 10000/14836\n",
      "Processing molecule 11000/14836\n",
      "Processing molecule 12000/14836\n",
      "Processing molecule 13000/14836\n",
      "Processing molecule 14000/14836\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14682\n",
      "Processing molecule 2000/14682\n",
      "Processing molecule 3000/14682\n",
      "Processing molecule 4000/14682\n",
      "Processing molecule 5000/14682\n",
      "Processing molecule 6000/14682\n",
      "Processing molecule 7000/14682\n",
      "Processing molecule 8000/14682\n",
      "Processing molecule 9000/14682\n",
      "Processing molecule 10000/14682\n",
      "Processing molecule 11000/14682\n",
      "Processing molecule 12000/14682\n",
      "Processing molecule 13000/14682\n",
      "Processing molecule 14000/14682\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14691\n",
      "Processing molecule 2000/14691\n",
      "Processing molecule 3000/14691\n",
      "Processing molecule 4000/14691\n",
      "Processing molecule 5000/14691\n",
      "Processing molecule 6000/14691\n",
      "Processing molecule 7000/14691\n",
      "Processing molecule 8000/14691\n",
      "Processing molecule 9000/14691\n",
      "Processing molecule 10000/14691\n",
      "Processing molecule 11000/14691\n",
      "Processing molecule 12000/14691\n",
      "Processing molecule 13000/14691\n",
      "Processing molecule 14000/14691\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14644\n",
      "Processing molecule 2000/14644\n",
      "Processing molecule 3000/14644\n",
      "Processing molecule 4000/14644\n",
      "Processing molecule 5000/14644\n",
      "Processing molecule 6000/14644\n",
      "Processing molecule 7000/14644\n",
      "Processing molecule 8000/14644\n",
      "Processing molecule 9000/14644\n",
      "Processing molecule 10000/14644\n",
      "Processing molecule 11000/14644\n",
      "Processing molecule 12000/14644\n",
      "Processing molecule 13000/14644\n",
      "Processing molecule 14000/14644\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14667\n",
      "Processing molecule 2000/14667\n",
      "Processing molecule 3000/14667\n",
      "Processing molecule 4000/14667\n",
      "Processing molecule 5000/14667\n",
      "Processing molecule 6000/14667\n",
      "Processing molecule 7000/14667\n",
      "Processing molecule 8000/14667\n",
      "Processing molecule 9000/14667\n",
      "Processing molecule 10000/14667\n",
      "Processing molecule 11000/14667\n",
      "Processing molecule 12000/14667\n",
      "Processing molecule 13000/14667\n",
      "Processing molecule 14000/14667\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14711\n",
      "Processing molecule 2000/14711\n",
      "Processing molecule 3000/14711\n",
      "Processing molecule 4000/14711\n",
      "Processing molecule 5000/14711\n",
      "Processing molecule 6000/14711\n",
      "Processing molecule 7000/14711\n",
      "Processing molecule 8000/14711\n",
      "Processing molecule 9000/14711\n",
      "Processing molecule 10000/14711\n",
      "Processing molecule 11000/14711\n",
      "Processing molecule 12000/14711\n",
      "Processing molecule 13000/14711\n",
      "Processing molecule 14000/14711\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14651\n",
      "Processing molecule 2000/14651\n",
      "Processing molecule 3000/14651\n",
      "Processing molecule 4000/14651\n",
      "Processing molecule 5000/14651\n",
      "Processing molecule 6000/14651\n",
      "Processing molecule 7000/14651\n",
      "Processing molecule 8000/14651\n",
      "Processing molecule 9000/14651\n",
      "Processing molecule 10000/14651\n",
      "Processing molecule 11000/14651\n",
      "Processing molecule 12000/14651\n",
      "Processing molecule 13000/14651\n",
      "Processing molecule 14000/14651\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14774\n",
      "Processing molecule 2000/14774\n",
      "Processing molecule 3000/14774\n",
      "Processing molecule 4000/14774\n",
      "Processing molecule 5000/14774\n",
      "Processing molecule 6000/14774\n",
      "Processing molecule 7000/14774\n",
      "Processing molecule 8000/14774\n",
      "Processing molecule 9000/14774\n",
      "Processing molecule 10000/14774\n",
      "Processing molecule 11000/14774\n",
      "Processing molecule 12000/14774\n",
      "Processing molecule 13000/14774\n",
      "Processing molecule 14000/14774\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14746\n",
      "Processing molecule 2000/14746\n",
      "Processing molecule 3000/14746\n",
      "Processing molecule 4000/14746\n",
      "Processing molecule 5000/14746\n",
      "Processing molecule 6000/14746\n",
      "Processing molecule 7000/14746\n",
      "Processing molecule 8000/14746\n",
      "Processing molecule 9000/14746\n",
      "Processing molecule 10000/14746\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing molecule 11000/14746\n",
      "Processing molecule 12000/14746\n",
      "Processing molecule 13000/14746\n",
      "Processing molecule 14000/14746\n",
      "Data created!!\n"
     ]
    }
   ],
   "source": [
    "data_ds = []\n",
    "\n",
    "for i, task in  enumerate(muv_tasks):\n",
    "    a = df[['smiles' , task]]\n",
    "    a = a.dropna()\n",
    "    ds = DATASET(a, smiles_to_bigraph, AttentiveFPAtomFeaturizer(), cache_file_path = cache_path)\n",
    "    data = create_dataset_with_gcn(ds, embed_class_muv, gcn_model, muv_tasks, i)\n",
    "    for d in data:\n",
    "        data_ds.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9wZRIKrq2Kec",
    "outputId": "6a5e45aa-32cb-47da-d25c-325f6cfe106b",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train positive label: 443 - train negative label: 224454\n",
      "up and down sampling => train positive label: 74867 - train negative label: 224454\n",
      "Test positive label: 46 - Test negative label: 24943\n",
      "Epoch 1/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5575 - accuracy: 0.7496 - mae: 0.3706 - mse: 0.1856 - auc_17: 0.5725\n",
      "Epoch 2/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5248 - accuracy: 0.7528 - mae: 0.3491 - mse: 0.1743 - auc_17: 0.6708\n",
      "Epoch 3/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5187 - accuracy: 0.7589 - mae: 0.3440 - mse: 0.1719 - auc_17: 0.6818\n",
      "Epoch 4/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5169 - accuracy: 0.7609 - mae: 0.3421 - mse: 0.1709 - auc_17: 0.6864\n",
      "Epoch 5/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5161 - accuracy: 0.7617 - mae: 0.3411 - mse: 0.1704 - auc_17: 0.6894\n",
      "Epoch 6/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5133 - accuracy: 0.7623 - mae: 0.3389 - mse: 0.1692 - auc_17: 0.6962\n",
      "Epoch 7/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5127 - accuracy: 0.7615 - mae: 0.3381 - mse: 0.1689 - auc_17: 0.6995\n",
      "Epoch 8/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5110 - accuracy: 0.7615 - mae: 0.3368 - mse: 0.1683 - auc_17: 0.7033\n",
      "Epoch 9/15\n",
      "2339/2339 [==============================] - 8s 4ms/step - loss: 0.5088 - accuracy: 0.7615 - mae: 0.3353 - mse: 0.1676 - auc_17: 0.7080\n",
      "Epoch 10/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5088 - accuracy: 0.7606 - mae: 0.3355 - mse: 0.1676 - auc_17: 0.7088\n",
      "Epoch 11/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5069 - accuracy: 0.7609 - mae: 0.3340 - mse: 0.1670 - auc_17: 0.7123\n",
      "Epoch 12/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5056 - accuracy: 0.7619 - mae: 0.3331 - mse: 0.1666 - auc_17: 0.7139\n",
      "Epoch 13/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5031 - accuracy: 0.7620 - mae: 0.3318 - mse: 0.1658 - auc_17: 0.7187\n",
      "Epoch 14/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5011 - accuracy: 0.7631 - mae: 0.3305 - mse: 0.1652 - auc_17: 0.7219\n",
      "Epoch 15/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5006 - accuracy: 0.7626 - mae: 0.3304 - mse: 0.1651 - auc_17: 0.7229\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "75\n",
      "74\n",
      "Epoch 1/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5006 - accuracy: 0.7628 - mae: 0.3303 - mse: 0.1651 - auc_17: 0.7228\n",
      "Epoch 2/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.5004 - accuracy: 0.7628 - mae: 0.3302 - mse: 0.1651 - auc_17: 0.7225\n",
      "Epoch 3/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.4985 - accuracy: 0.7621 - mae: 0.3294 - mse: 0.1646 - auc_17: 0.7259\n",
      "Epoch 4/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.4969 - accuracy: 0.7636 - mae: 0.3281 - mse: 0.1640 - auc_17: 0.7282\n",
      "Epoch 5/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.4952 - accuracy: 0.7635 - mae: 0.3270 - mse: 0.1635 - auc_17: 0.7316\n",
      "Epoch 6/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.4954 - accuracy: 0.7633 - mae: 0.3272 - mse: 0.1636 - auc_17: 0.7315\n",
      "Epoch 7/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.4972 - accuracy: 0.7636 - mae: 0.3284 - mse: 0.1641 - auc_17: 0.7280\n",
      "Epoch 8/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.4957 - accuracy: 0.7638 - mae: 0.3276 - mse: 0.1637 - auc_17: 0.7297\n",
      "Epoch 9/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.4943 - accuracy: 0.7647 - mae: 0.3264 - mse: 0.1631 - auc_17: 0.7326\n",
      "Epoch 10/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.4931 - accuracy: 0.7652 - mae: 0.3254 - mse: 0.1627 - auc_17: 0.7343\n",
      "Epoch 11/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.4930 - accuracy: 0.7657 - mae: 0.3254 - mse: 0.1626 - auc_17: 0.7343\n",
      "Epoch 12/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.4920 - accuracy: 0.7653 - mae: 0.3250 - mse: 0.1625 - auc_17: 0.7360\n",
      "Epoch 13/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.4892 - accuracy: 0.7665 - mae: 0.3232 - mse: 0.1615 - auc_17: 0.7407\n",
      "Epoch 14/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.4883 - accuracy: 0.7671 - mae: 0.3224 - mse: 0.1612 - auc_17: 0.7413\n",
      "Epoch 15/15\n",
      "2339/2339 [==============================] - 9s 4ms/step - loss: 0.4889 - accuracy: 0.7684 - mae: 0.3226 - mse: 0.1613 - auc_17: 0.7397\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "2\n",
      "781/781 [==============================] - 1s 751us/step - loss: 0.2230 - accuracy: 0.9800 - mae: 0.1875 - mse: 0.0512 - auc_17: 0.6839\n",
      "train positive label: 444 - train negative label: 224453\n",
      "up and down sampling => train positive label: 75036 - train negative label: 224453\n",
      "Test positive label: 45 - Test negative label: 24944\n",
      "Epoch 1/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5517 - accuracy: 0.7435 - mae: 0.3688 - mse: 0.1841 - auc_18: 0.6090\n",
      "Epoch 2/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5223 - accuracy: 0.7571 - mae: 0.3464 - mse: 0.1728 - auc_18: 0.6785\n",
      "Epoch 3/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5179 - accuracy: 0.7602 - mae: 0.3428 - mse: 0.1712 - auc_18: 0.6863\n",
      "Epoch 4/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5147 - accuracy: 0.7622 - mae: 0.3402 - mse: 0.1700 - auc_18: 0.6917\n",
      "Epoch 5/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5128 - accuracy: 0.7627 - mae: 0.3387 - mse: 0.1693 - auc_18: 0.6959\n",
      "Epoch 6/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5113 - accuracy: 0.7632 - mae: 0.3377 - mse: 0.1687 - auc_18: 0.6982\n",
      "Epoch 7/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5086 - accuracy: 0.7639 - mae: 0.3358 - mse: 0.1679 - auc_18: 0.7026\n",
      "Epoch 8/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5077 - accuracy: 0.7634 - mae: 0.3353 - mse: 0.1676 - auc_18: 0.7049\n",
      "Epoch 9/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5048 - accuracy: 0.7645 - mae: 0.3335 - mse: 0.1666 - auc_18: 0.7102\n",
      "Epoch 10/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5035 - accuracy: 0.7647 - mae: 0.3324 - mse: 0.1661 - auc_18: 0.7123\n",
      "Epoch 11/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5056 - accuracy: 0.7625 - mae: 0.3343 - mse: 0.1670 - auc_18: 0.7105\n",
      "Epoch 12/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5040 - accuracy: 0.7624 - mae: 0.3330 - mse: 0.1665 - auc_18: 0.7133\n",
      "Epoch 13/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5029 - accuracy: 0.7624 - mae: 0.3323 - mse: 0.1661 - auc_18: 0.7161\n",
      "Epoch 14/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.5004 - accuracy: 0.7636 - mae: 0.3306 - mse: 0.1652 - auc_18: 0.7196\n",
      "Epoch 15/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4987 - accuracy: 0.7652 - mae: 0.3293 - mse: 0.1646 - auc_18: 0.7229\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "74\n",
      "Epoch 1/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4968 - accuracy: 0.7650 - mae: 0.3282 - mse: 0.1641 - auc_18: 0.7253\n",
      "Epoch 2/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4965 - accuracy: 0.7662 - mae: 0.3282 - mse: 0.1639 - auc_18: 0.7253\n",
      "Epoch 3/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4991 - accuracy: 0.7651 - mae: 0.3295 - mse: 0.1647 - auc_18: 0.7223\n",
      "Epoch 4/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4983 - accuracy: 0.7660 - mae: 0.3286 - mse: 0.1643 - auc_18: 0.7233\n",
      "Epoch 5/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4962 - accuracy: 0.7661 - mae: 0.3277 - mse: 0.1637 - auc_18: 0.7269\n",
      "Epoch 6/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4955 - accuracy: 0.7664 - mae: 0.3269 - mse: 0.1633 - auc_18: 0.7291\n",
      "Epoch 7/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4924 - accuracy: 0.7675 - mae: 0.3248 - mse: 0.1623 - auc_18: 0.7338\n",
      "Epoch 8/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4899 - accuracy: 0.7687 - mae: 0.3228 - mse: 0.1614 - auc_18: 0.7382\n",
      "Epoch 9/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4884 - accuracy: 0.7693 - mae: 0.3218 - mse: 0.1608 - auc_18: 0.7408\n",
      "Epoch 10/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4895 - accuracy: 0.7695 - mae: 0.3223 - mse: 0.1611 - auc_18: 0.7390\n",
      "Epoch 11/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4880 - accuracy: 0.7695 - mae: 0.3213 - mse: 0.1607 - auc_18: 0.7417\n",
      "Epoch 12/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4888 - accuracy: 0.7700 - mae: 0.3222 - mse: 0.1610 - auc_18: 0.7386\n",
      "Epoch 13/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4903 - accuracy: 0.7702 - mae: 0.3229 - mse: 0.1614 - auc_18: 0.7361\n",
      "Epoch 14/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4864 - accuracy: 0.7712 - mae: 0.3205 - mse: 0.1602 - auc_18: 0.7426\n",
      "Epoch 15/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4847 - accuracy: 0.7714 - mae: 0.3194 - mse: 0.1596 - auc_18: 0.7454\n",
      "77\n",
      "77\n",
      "77\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "Epoch 1/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4834 - accuracy: 0.7715 - mae: 0.3187 - mse: 0.1593 - auc_18: 0.7473\n",
      "Epoch 2/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4833 - accuracy: 0.7718 - mae: 0.3183 - mse: 0.1591 - auc_18: 0.7473\n",
      "Epoch 3/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4808 - accuracy: 0.7724 - mae: 0.3171 - mse: 0.1584 - auc_18: 0.7508\n",
      "Epoch 4/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4808 - accuracy: 0.7732 - mae: 0.3167 - mse: 0.1584 - auc_18: 0.7506\n",
      "Epoch 5/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4794 - accuracy: 0.7738 - mae: 0.3159 - mse: 0.1579 - auc_18: 0.7522\n",
      "Epoch 6/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4800 - accuracy: 0.7727 - mae: 0.3164 - mse: 0.1582 - auc_18: 0.7516\n",
      "Epoch 7/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4768 - accuracy: 0.7745 - mae: 0.3140 - mse: 0.1570 - auc_18: 0.7568\n",
      "Epoch 8/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4768 - accuracy: 0.7740 - mae: 0.3143 - mse: 0.1570 - auc_18: 0.7566\n",
      "Epoch 9/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4762 - accuracy: 0.7743 - mae: 0.3139 - mse: 0.1568 - auc_18: 0.7572\n",
      "Epoch 10/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4761 - accuracy: 0.7747 - mae: 0.3139 - mse: 0.1569 - auc_18: 0.7569\n",
      "Epoch 11/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4739 - accuracy: 0.7754 - mae: 0.3121 - mse: 0.1560 - auc_18: 0.7606\n",
      "Epoch 12/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4736 - accuracy: 0.7751 - mae: 0.3121 - mse: 0.1561 - auc_18: 0.7610\n",
      "Epoch 13/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4767 - accuracy: 0.7749 - mae: 0.3140 - mse: 0.1569 - auc_18: 0.7553\n",
      "Epoch 14/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4731 - accuracy: 0.7764 - mae: 0.3116 - mse: 0.1558 - auc_18: 0.7611\n",
      "Epoch 15/15\n",
      "2340/2340 [==============================] - 8s 3ms/step - loss: 0.4717 - accuracy: 0.7772 - mae: 0.3110 - mse: 0.1554 - auc_18: 0.7630\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "3\n",
      "781/781 [==============================] - 1s 732us/step - loss: 0.2376 - accuracy: 0.9724 - mae: 0.1981 - mse: 0.0575 - auc_18: 0.6760\n",
      "train positive label: 434 - train negative label: 224463\n",
      "up and down sampling => train positive label: 75082 - train negative label: 224463\n",
      "Test positive label: 55 - Test negative label: 24934\n",
      "Epoch 1/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5551 - accuracy: 0.7489 - mae: 0.3701 - mse: 0.1849 - auc_19: 0.5846\n",
      "Epoch 2/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5252 - accuracy: 0.7504 - mae: 0.3487 - mse: 0.1740 - auc_19: 0.6769\n",
      "Epoch 3/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5189 - accuracy: 0.7602 - mae: 0.3434 - mse: 0.1713 - auc_19: 0.6861\n",
      "Epoch 4/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5153 - accuracy: 0.7618 - mae: 0.3404 - mse: 0.1700 - auc_19: 0.6927\n",
      "Epoch 5/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5135 - accuracy: 0.7627 - mae: 0.3387 - mse: 0.1692 - auc_19: 0.6962\n",
      "Epoch 6/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5133 - accuracy: 0.7628 - mae: 0.3380 - mse: 0.1689 - auc_19: 0.6979\n",
      "Epoch 7/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5108 - accuracy: 0.7645 - mae: 0.3362 - mse: 0.1679 - auc_19: 0.7027\n",
      "Epoch 8/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5080 - accuracy: 0.7650 - mae: 0.3343 - mse: 0.1669 - auc_19: 0.7081\n",
      "Epoch 9/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5055 - accuracy: 0.7641 - mae: 0.3323 - mse: 0.1661 - auc_19: 0.7129\n",
      "Epoch 10/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5024 - accuracy: 0.7649 - mae: 0.3302 - mse: 0.1650 - auc_19: 0.7187\n",
      "Epoch 11/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5007 - accuracy: 0.7660 - mae: 0.3290 - mse: 0.1644 - auc_19: 0.7212\n",
      "Epoch 12/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4983 - accuracy: 0.7667 - mae: 0.3272 - mse: 0.1635 - auc_19: 0.7248\n",
      "Epoch 13/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5007 - accuracy: 0.7667 - mae: 0.3288 - mse: 0.1643 - auc_19: 0.7194\n",
      "Epoch 14/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4990 - accuracy: 0.7673 - mae: 0.3277 - mse: 0.1638 - auc_19: 0.7225\n",
      "Epoch 15/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4998 - accuracy: 0.7680 - mae: 0.3283 - mse: 0.1641 - auc_19: 0.7204\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "74\n",
      "Epoch 1/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5030 - accuracy: 0.7652 - mae: 0.3307 - mse: 0.1653 - auc_19: 0.7168\n",
      "Epoch 2/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4996 - accuracy: 0.7664 - mae: 0.3284 - mse: 0.1641 - auc_19: 0.7230\n",
      "Epoch 3/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4975 - accuracy: 0.7684 - mae: 0.3268 - mse: 0.1634 - auc_19: 0.7258\n",
      "Epoch 4/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4955 - accuracy: 0.7688 - mae: 0.3255 - mse: 0.1627 - auc_19: 0.7295\n",
      "Epoch 5/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4946 - accuracy: 0.7691 - mae: 0.3248 - mse: 0.1623 - auc_19: 0.7313\n",
      "Epoch 6/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4914 - accuracy: 0.7704 - mae: 0.3226 - mse: 0.1613 - auc_19: 0.7359\n",
      "Epoch 7/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4907 - accuracy: 0.7710 - mae: 0.3221 - mse: 0.1611 - auc_19: 0.7366\n",
      "Epoch 8/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4884 - accuracy: 0.7717 - mae: 0.3207 - mse: 0.1603 - auc_19: 0.7402\n",
      "Epoch 9/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4876 - accuracy: 0.7715 - mae: 0.3200 - mse: 0.1600 - auc_19: 0.7417\n",
      "Epoch 10/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4848 - accuracy: 0.7726 - mae: 0.3182 - mse: 0.1590 - auc_19: 0.7464\n",
      "Epoch 11/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4836 - accuracy: 0.7728 - mae: 0.3176 - mse: 0.1587 - auc_19: 0.7482\n",
      "Epoch 12/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4816 - accuracy: 0.7737 - mae: 0.3161 - mse: 0.1580 - auc_19: 0.7514\n",
      "Epoch 13/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4805 - accuracy: 0.7746 - mae: 0.3151 - mse: 0.1575 - auc_19: 0.7529\n",
      "Epoch 14/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4790 - accuracy: 0.7745 - mae: 0.3144 - mse: 0.1571 - auc_19: 0.7553\n",
      "Epoch 15/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4782 - accuracy: 0.7748 - mae: 0.3139 - mse: 0.1569 - auc_19: 0.7566\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "Epoch 1/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4767 - accuracy: 0.7762 - mae: 0.3129 - mse: 0.1564 - auc_19: 0.7582\n",
      "Epoch 2/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4747 - accuracy: 0.7765 - mae: 0.3112 - mse: 0.1557 - auc_19: 0.7615\n",
      "Epoch 3/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4733 - accuracy: 0.7760 - mae: 0.3106 - mse: 0.1553 - auc_19: 0.7639\n",
      "Epoch 4/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4719 - accuracy: 0.7768 - mae: 0.3098 - mse: 0.1548 - auc_19: 0.7657\n",
      "Epoch 5/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4720 - accuracy: 0.7775 - mae: 0.3096 - mse: 0.1548 - auc_19: 0.7654\n",
      "Epoch 6/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4699 - accuracy: 0.7775 - mae: 0.3083 - mse: 0.1542 - auc_19: 0.7684\n",
      "Epoch 7/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4687 - accuracy: 0.7774 - mae: 0.3078 - mse: 0.1538 - auc_19: 0.7702\n",
      "Epoch 8/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4677 - accuracy: 0.7784 - mae: 0.3071 - mse: 0.1535 - auc_19: 0.7717\n",
      "Epoch 9/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4678 - accuracy: 0.7785 - mae: 0.3071 - mse: 0.1535 - auc_19: 0.7709\n",
      "Epoch 10/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4668 - accuracy: 0.7790 - mae: 0.3063 - mse: 0.1532 - auc_19: 0.7725\n",
      "Epoch 11/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4658 - accuracy: 0.7792 - mae: 0.3059 - mse: 0.1529 - auc_19: 0.7739\n",
      "Epoch 12/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4670 - accuracy: 0.7792 - mae: 0.3061 - mse: 0.1530 - auc_19: 0.7728\n",
      "Epoch 13/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4644 - accuracy: 0.7808 - mae: 0.3043 - mse: 0.1521 - auc_19: 0.7762\n",
      "Epoch 14/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4633 - accuracy: 0.7811 - mae: 0.3037 - mse: 0.1519 - auc_19: 0.7772\n",
      "Epoch 15/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4610 - accuracy: 0.7818 - mae: 0.3024 - mse: 0.1511 - auc_19: 0.7805\n",
      "78\n",
      "78\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "Epoch 1/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4609 - accuracy: 0.7818 - mae: 0.3023 - mse: 0.1512 - auc_19: 0.7805\n",
      "Epoch 2/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4590 - accuracy: 0.7829 - mae: 0.3007 - mse: 0.1503 - auc_19: 0.7828\n",
      "Epoch 3/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4577 - accuracy: 0.7829 - mae: 0.3003 - mse: 0.1501 - auc_19: 0.7848\n",
      "Epoch 4/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4562 - accuracy: 0.7840 - mae: 0.2990 - mse: 0.1495 - auc_19: 0.7863\n",
      "Epoch 5/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4561 - accuracy: 0.7842 - mae: 0.2989 - mse: 0.1494 - auc_19: 0.7866\n",
      "Epoch 6/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4559 - accuracy: 0.7837 - mae: 0.2989 - mse: 0.1494 - auc_19: 0.7866\n",
      "Epoch 7/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4581 - accuracy: 0.7835 - mae: 0.3004 - mse: 0.1501 - auc_19: 0.7834\n",
      "Epoch 8/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4571 - accuracy: 0.7827 - mae: 0.2998 - mse: 0.1499 - auc_19: 0.7853\n",
      "Epoch 9/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4541 - accuracy: 0.7846 - mae: 0.2975 - mse: 0.1487 - auc_19: 0.7889\n",
      "Epoch 10/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4537 - accuracy: 0.7848 - mae: 0.2972 - mse: 0.1486 - auc_19: 0.7899\n",
      "Epoch 11/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4538 - accuracy: 0.7845 - mae: 0.2971 - mse: 0.1486 - auc_19: 0.7892\n",
      "Epoch 12/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4549 - accuracy: 0.7839 - mae: 0.2982 - mse: 0.1489 - auc_19: 0.7888\n",
      "Epoch 13/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4517 - accuracy: 0.7847 - mae: 0.2960 - mse: 0.1480 - auc_19: 0.7918\n",
      "Epoch 14/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4521 - accuracy: 0.7843 - mae: 0.2965 - mse: 0.1482 - auc_19: 0.7919\n",
      "Epoch 15/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4513 - accuracy: 0.7851 - mae: 0.2956 - mse: 0.1478 - auc_19: 0.7928\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "4\n",
      "781/781 [==============================] - 1s 870us/step - loss: 0.3201 - accuracy: 0.9284 - mae: 0.2487 - mse: 0.0905 - auc_19: 0.6512\n",
      "train positive label: 444 - train negative label: 224453\n",
      "up and down sampling => train positive label: 75036 - train negative label: 224453\n",
      "Test positive label: 45 - Test negative label: 24944\n",
      "Epoch 1/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.5519 - accuracy: 0.7444 - mae: 0.3689 - mse: 0.1843 - auc_20: 0.6129\n",
      "Epoch 2/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.5210 - accuracy: 0.7566 - mae: 0.3452 - mse: 0.1723 - auc_20: 0.6835\n",
      "Epoch 3/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.5158 - accuracy: 0.7600 - mae: 0.3415 - mse: 0.1704 - auc_20: 0.6933\n",
      "Epoch 4/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.5124 - accuracy: 0.7613 - mae: 0.3390 - mse: 0.1692 - auc_20: 0.6994\n",
      "Epoch 5/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.5096 - accuracy: 0.7620 - mae: 0.3367 - mse: 0.1681 - auc_20: 0.7050\n",
      "Epoch 6/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.5060 - accuracy: 0.7637 - mae: 0.3340 - mse: 0.1668 - auc_20: 0.7105\n",
      "Epoch 7/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.5031 - accuracy: 0.7650 - mae: 0.3321 - mse: 0.1658 - auc_20: 0.7147\n",
      "Epoch 8/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.5002 - accuracy: 0.7664 - mae: 0.3300 - mse: 0.1649 - auc_20: 0.7194\n",
      "Epoch 9/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4987 - accuracy: 0.7663 - mae: 0.3291 - mse: 0.1644 - auc_20: 0.7220\n",
      "Epoch 10/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4966 - accuracy: 0.7679 - mae: 0.3275 - mse: 0.1636 - auc_20: 0.7250\n",
      "Epoch 11/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4962 - accuracy: 0.7684 - mae: 0.3272 - mse: 0.1634 - auc_20: 0.7258\n",
      "Epoch 12/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4952 - accuracy: 0.7690 - mae: 0.3263 - mse: 0.1630 - auc_20: 0.7276\n",
      "Epoch 13/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4928 - accuracy: 0.7699 - mae: 0.3245 - mse: 0.1621 - auc_20: 0.7316\n",
      "Epoch 14/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4897 - accuracy: 0.7708 - mae: 0.3226 - mse: 0.1611 - auc_20: 0.7369\n",
      "Epoch 15/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4864 - accuracy: 0.7731 - mae: 0.3198 - mse: 0.1599 - auc_20: 0.7415\n",
      "77\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "74\n",
      "Epoch 1/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4859 - accuracy: 0.7733 - mae: 0.3191 - mse: 0.1595 - auc_20: 0.7429\n",
      "Epoch 2/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4839 - accuracy: 0.7744 - mae: 0.3176 - mse: 0.1587 - auc_20: 0.7463\n",
      "Epoch 3/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4822 - accuracy: 0.7760 - mae: 0.3160 - mse: 0.1579 - auc_20: 0.7486\n",
      "Epoch 4/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4794 - accuracy: 0.7766 - mae: 0.3143 - mse: 0.1571 - auc_20: 0.7528\n",
      "Epoch 5/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4787 - accuracy: 0.7765 - mae: 0.3136 - mse: 0.1568 - auc_20: 0.7545\n",
      "Epoch 6/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4833 - accuracy: 0.7757 - mae: 0.3168 - mse: 0.1583 - auc_20: 0.7477\n",
      "Epoch 7/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4856 - accuracy: 0.7750 - mae: 0.3183 - mse: 0.1591 - auc_20: 0.7434\n",
      "Epoch 8/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4834 - accuracy: 0.7763 - mae: 0.3165 - mse: 0.1583 - auc_20: 0.7464\n",
      "Epoch 9/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4813 - accuracy: 0.7765 - mae: 0.3154 - mse: 0.1576 - auc_20: 0.7501\n",
      "Epoch 10/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4792 - accuracy: 0.7770 - mae: 0.3143 - mse: 0.1570 - auc_20: 0.7531\n",
      "Epoch 11/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4785 - accuracy: 0.7769 - mae: 0.3135 - mse: 0.1568 - auc_20: 0.7547\n",
      "Epoch 12/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4767 - accuracy: 0.7771 - mae: 0.3123 - mse: 0.1561 - auc_20: 0.7578\n",
      "Epoch 13/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4753 - accuracy: 0.7772 - mae: 0.3117 - mse: 0.1558 - auc_20: 0.7596\n",
      "Epoch 14/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4765 - accuracy: 0.7765 - mae: 0.3124 - mse: 0.1563 - auc_20: 0.7584\n",
      "Epoch 15/15\n",
      "2340/2340 [==============================] - 9s 4ms/step - loss: 0.4736 - accuracy: 0.7777 - mae: 0.3106 - mse: 0.1553 - auc_20: 0.7625\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "2\n",
      "781/781 [==============================] - 1s 778us/step - loss: 0.3018 - accuracy: 0.9422 - mae: 0.2434 - mse: 0.0811 - auc_20: 0.6854\n",
      "train positive label: 432 - train negative label: 224465\n",
      "up and down sampling => train positive label: 75168 - train negative label: 224465\n",
      "Test positive label: 57 - Test negative label: 24932\n",
      "Epoch 1/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5663 - accuracy: 0.7467 - mae: 0.3779 - mse: 0.1892 - auc_21: 0.5324\n",
      "Epoch 2/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5303 - accuracy: 0.7520 - mae: 0.3521 - mse: 0.1760 - auc_21: 0.6628\n",
      "Epoch 3/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5214 - accuracy: 0.7572 - mae: 0.3456 - mse: 0.1726 - auc_21: 0.6800\n",
      "Epoch 4/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5171 - accuracy: 0.7604 - mae: 0.3421 - mse: 0.1709 - auc_21: 0.6890\n",
      "Epoch 5/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5130 - accuracy: 0.7623 - mae: 0.3391 - mse: 0.1694 - auc_21: 0.6965\n",
      "Epoch 6/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5109 - accuracy: 0.7631 - mae: 0.3375 - mse: 0.1686 - auc_21: 0.7002\n",
      "Epoch 7/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5107 - accuracy: 0.7632 - mae: 0.3372 - mse: 0.1685 - auc_21: 0.7003\n",
      "Epoch 8/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5088 - accuracy: 0.7637 - mae: 0.3360 - mse: 0.1679 - auc_21: 0.7023\n",
      "Epoch 9/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5076 - accuracy: 0.7650 - mae: 0.3351 - mse: 0.1674 - auc_21: 0.7049\n",
      "Epoch 10/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5065 - accuracy: 0.7649 - mae: 0.3342 - mse: 0.1670 - auc_21: 0.7066\n",
      "Epoch 11/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5048 - accuracy: 0.7665 - mae: 0.3327 - mse: 0.1663 - auc_21: 0.7095\n",
      "Epoch 12/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5041 - accuracy: 0.7666 - mae: 0.3321 - mse: 0.1660 - auc_21: 0.7107\n",
      "Epoch 13/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5024 - accuracy: 0.7665 - mae: 0.3311 - mse: 0.1655 - auc_21: 0.7140\n",
      "Epoch 14/15\n",
      "2341/2341 [==============================] - 8s 3ms/step - loss: 0.5028 - accuracy: 0.7662 - mae: 0.3313 - mse: 0.1656 - auc_21: 0.7127\n",
      "Epoch 15/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.5042 - accuracy: 0.7666 - mae: 0.3323 - mse: 0.1661 - auc_21: 0.7092\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "75\n",
      "74\n",
      "Epoch 1/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.5046 - accuracy: 0.7672 - mae: 0.3318 - mse: 0.1658 - auc_21: 0.7110\n",
      "Epoch 2/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.5042 - accuracy: 0.7674 - mae: 0.3314 - mse: 0.1656 - auc_21: 0.7125\n",
      "Epoch 3/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.5047 - accuracy: 0.7674 - mae: 0.3312 - mse: 0.1656 - auc_21: 0.7129\n",
      "Epoch 4/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.5061 - accuracy: 0.7655 - mae: 0.3324 - mse: 0.1661 - auc_21: 0.7119\n",
      "Epoch 5/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.5058 - accuracy: 0.7656 - mae: 0.3321 - mse: 0.1660 - auc_21: 0.7129\n",
      "Epoch 6/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.5043 - accuracy: 0.7664 - mae: 0.3308 - mse: 0.1655 - auc_21: 0.7150\n",
      "Epoch 7/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.5018 - accuracy: 0.7670 - mae: 0.3294 - mse: 0.1646 - auc_21: 0.7197\n",
      "Epoch 8/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.5005 - accuracy: 0.7676 - mae: 0.3283 - mse: 0.1642 - auc_21: 0.7216\n",
      "Epoch 9/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4984 - accuracy: 0.7687 - mae: 0.3268 - mse: 0.1634 - auc_21: 0.7259\n",
      "Epoch 10/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4956 - accuracy: 0.7701 - mae: 0.3248 - mse: 0.1624 - auc_21: 0.7299\n",
      "Epoch 11/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4954 - accuracy: 0.7703 - mae: 0.3246 - mse: 0.1623 - auc_21: 0.7306\n",
      "Epoch 12/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4959 - accuracy: 0.7693 - mae: 0.3251 - mse: 0.1625 - auc_21: 0.7296\n",
      "Epoch 13/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4936 - accuracy: 0.7710 - mae: 0.3235 - mse: 0.1617 - auc_21: 0.7331\n",
      "Epoch 14/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4917 - accuracy: 0.7707 - mae: 0.3222 - mse: 0.1611 - auc_21: 0.7363\n",
      "Epoch 15/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4916 - accuracy: 0.7707 - mae: 0.3222 - mse: 0.1611 - auc_21: 0.7370\n",
      "77\n",
      "77\n",
      "76\n",
      "77\n",
      "77\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "Epoch 1/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4920 - accuracy: 0.7690 - mae: 0.3231 - mse: 0.1615 - auc_21: 0.7374\n",
      "Epoch 2/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4901 - accuracy: 0.7705 - mae: 0.3213 - mse: 0.1607 - auc_21: 0.7399\n",
      "Epoch 3/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4893 - accuracy: 0.7701 - mae: 0.3210 - mse: 0.1605 - auc_21: 0.7419\n",
      "Epoch 4/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4887 - accuracy: 0.7716 - mae: 0.3203 - mse: 0.1601 - auc_21: 0.7423\n",
      "Epoch 5/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4883 - accuracy: 0.7717 - mae: 0.3202 - mse: 0.1601 - auc_21: 0.7431\n",
      "Epoch 6/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4868 - accuracy: 0.7710 - mae: 0.3193 - mse: 0.1597 - auc_21: 0.7456\n",
      "Epoch 7/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4858 - accuracy: 0.7727 - mae: 0.3184 - mse: 0.1592 - auc_21: 0.7466\n",
      "Epoch 8/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4850 - accuracy: 0.7723 - mae: 0.3180 - mse: 0.1590 - auc_21: 0.7482\n",
      "Epoch 9/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4843 - accuracy: 0.7729 - mae: 0.3174 - mse: 0.1587 - auc_21: 0.7493\n",
      "Epoch 10/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4825 - accuracy: 0.7739 - mae: 0.3162 - mse: 0.1581 - auc_21: 0.7517\n",
      "Epoch 11/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4828 - accuracy: 0.7727 - mae: 0.3166 - mse: 0.1583 - auc_21: 0.7517\n",
      "Epoch 12/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4802 - accuracy: 0.7743 - mae: 0.3146 - mse: 0.1573 - auc_21: 0.7558\n",
      "Epoch 13/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4796 - accuracy: 0.7744 - mae: 0.3144 - mse: 0.1571 - auc_21: 0.7563\n",
      "Epoch 14/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4774 - accuracy: 0.7749 - mae: 0.3131 - mse: 0.1566 - auc_21: 0.7594\n",
      "Epoch 15/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4774 - accuracy: 0.7751 - mae: 0.3131 - mse: 0.1565 - auc_21: 0.7597\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "76\n",
      "Epoch 1/15\n",
      "2341/2341 [==============================] - 9s 4ms/step - loss: 0.4766 - accuracy: 0.7752 - mae: 0.3125 - mse: 0.1562 - auc_21: 0.7613\n",
      "Epoch 2/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4770 - accuracy: 0.7749 - mae: 0.3130 - mse: 0.1565 - auc_21: 0.7597\n",
      "Epoch 3/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4749 - accuracy: 0.7762 - mae: 0.3113 - mse: 0.1556 - auc_21: 0.7631\n",
      "Epoch 4/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4737 - accuracy: 0.7763 - mae: 0.3103 - mse: 0.1551 - auc_21: 0.7649\n",
      "Epoch 5/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4773 - accuracy: 0.7757 - mae: 0.3127 - mse: 0.1563 - auc_21: 0.7596\n",
      "Epoch 6/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4733 - accuracy: 0.7765 - mae: 0.3102 - mse: 0.1551 - auc_21: 0.7657\n",
      "Epoch 7/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4713 - accuracy: 0.7774 - mae: 0.3088 - mse: 0.1544 - auc_21: 0.7685\n",
      "Epoch 8/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4725 - accuracy: 0.7761 - mae: 0.3100 - mse: 0.1549 - auc_21: 0.7669\n",
      "Epoch 9/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4698 - accuracy: 0.7776 - mae: 0.3079 - mse: 0.1540 - auc_21: 0.7704\n",
      "Epoch 10/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4697 - accuracy: 0.7775 - mae: 0.3078 - mse: 0.1539 - auc_21: 0.7711\n",
      "Epoch 11/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4689 - accuracy: 0.7783 - mae: 0.3070 - mse: 0.1535 - auc_21: 0.7720\n",
      "Epoch 12/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4679 - accuracy: 0.7789 - mae: 0.3068 - mse: 0.1533 - auc_21: 0.7729\n",
      "Epoch 13/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4681 - accuracy: 0.7783 - mae: 0.3069 - mse: 0.1534 - auc_21: 0.7725\n",
      "Epoch 14/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4700 - accuracy: 0.7781 - mae: 0.3079 - mse: 0.1539 - auc_21: 0.7705\n",
      "Epoch 15/15\n",
      "2341/2341 [==============================] - 8s 4ms/step - loss: 0.4686 - accuracy: 0.7789 - mae: 0.3070 - mse: 0.1535 - auc_21: 0.7722\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "4\n",
      "781/781 [==============================] - 0s 613us/step - loss: 0.2184 - accuracy: 0.9685 - mae: 0.1827 - mse: 0.0511 - auc_21: 0.6881\n",
      "train positive label: 445 - train negative label: 224452\n",
      "up and down sampling => train positive label: 75205 - train negative label: 224452\n",
      "Test positive label: 44 - Test negative label: 24945\n",
      "Epoch 1/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5538 - accuracy: 0.7465 - mae: 0.3693 - mse: 0.1847 - auc_22: 0.5957\n",
      "Epoch 2/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5207 - accuracy: 0.7573 - mae: 0.3451 - mse: 0.1723 - auc_22: 0.6826\n",
      "Epoch 3/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5149 - accuracy: 0.7613 - mae: 0.3406 - mse: 0.1701 - auc_22: 0.6925\n",
      "Epoch 4/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5109 - accuracy: 0.7629 - mae: 0.3378 - mse: 0.1687 - auc_22: 0.6994\n",
      "Epoch 5/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5081 - accuracy: 0.7631 - mae: 0.3361 - mse: 0.1678 - auc_22: 0.7049\n",
      "Epoch 6/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5049 - accuracy: 0.7640 - mae: 0.3337 - mse: 0.1667 - auc_22: 0.7116\n",
      "Epoch 7/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5015 - accuracy: 0.7646 - mae: 0.3308 - mse: 0.1653 - auc_22: 0.7196\n",
      "Epoch 8/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4990 - accuracy: 0.7644 - mae: 0.3291 - mse: 0.1644 - auc_22: 0.7250\n",
      "Epoch 9/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4973 - accuracy: 0.7656 - mae: 0.3278 - mse: 0.1638 - auc_22: 0.7273\n",
      "Epoch 10/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4960 - accuracy: 0.7655 - mae: 0.3266 - mse: 0.1633 - auc_22: 0.7295\n",
      "Epoch 11/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4963 - accuracy: 0.7656 - mae: 0.3272 - mse: 0.1635 - auc_22: 0.7291\n",
      "Epoch 12/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4964 - accuracy: 0.7663 - mae: 0.3269 - mse: 0.1634 - auc_22: 0.7287\n",
      "Epoch 13/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4958 - accuracy: 0.7656 - mae: 0.3268 - mse: 0.1634 - auc_22: 0.7296\n",
      "Epoch 14/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4924 - accuracy: 0.7670 - mae: 0.3245 - mse: 0.1623 - auc_22: 0.7348\n",
      "Epoch 15/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4899 - accuracy: 0.7681 - mae: 0.3230 - mse: 0.1614 - auc_22: 0.7384\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "74\n",
      "Epoch 1/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4869 - accuracy: 0.7687 - mae: 0.3207 - mse: 0.1604 - auc_22: 0.7434\n",
      "Epoch 2/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4872 - accuracy: 0.7690 - mae: 0.3210 - mse: 0.1604 - auc_22: 0.7429\n",
      "Epoch 3/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4876 - accuracy: 0.7683 - mae: 0.3216 - mse: 0.1607 - auc_22: 0.7423\n",
      "Epoch 4/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4842 - accuracy: 0.7688 - mae: 0.3194 - mse: 0.1597 - auc_22: 0.7475\n",
      "Epoch 5/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4823 - accuracy: 0.7705 - mae: 0.3182 - mse: 0.1590 - auc_22: 0.7497\n",
      "Epoch 6/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4832 - accuracy: 0.7702 - mae: 0.3186 - mse: 0.1593 - auc_22: 0.7482\n",
      "Epoch 7/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4807 - accuracy: 0.7710 - mae: 0.3171 - mse: 0.1584 - auc_22: 0.7524\n",
      "Epoch 8/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4805 - accuracy: 0.7713 - mae: 0.3168 - mse: 0.1583 - auc_22: 0.7521\n",
      "Epoch 9/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4786 - accuracy: 0.7711 - mae: 0.3156 - mse: 0.1578 - auc_22: 0.7557\n",
      "Epoch 10/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4764 - accuracy: 0.7722 - mae: 0.3142 - mse: 0.1570 - auc_22: 0.7587\n",
      "Epoch 11/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4754 - accuracy: 0.7716 - mae: 0.3137 - mse: 0.1569 - auc_22: 0.7598\n",
      "Epoch 12/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4744 - accuracy: 0.7722 - mae: 0.3130 - mse: 0.1564 - auc_22: 0.7616\n",
      "Epoch 13/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4726 - accuracy: 0.7730 - mae: 0.3115 - mse: 0.1558 - auc_22: 0.7643\n",
      "Epoch 14/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4719 - accuracy: 0.7727 - mae: 0.3113 - mse: 0.1555 - auc_22: 0.7653\n",
      "Epoch 15/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4719 - accuracy: 0.7715 - mae: 0.3114 - mse: 0.1557 - auc_22: 0.7657\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "Epoch 1/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4724 - accuracy: 0.7728 - mae: 0.3117 - mse: 0.1557 - auc_22: 0.7642\n",
      "Epoch 2/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4724 - accuracy: 0.7725 - mae: 0.3117 - mse: 0.1558 - auc_22: 0.7643\n",
      "Epoch 3/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4713 - accuracy: 0.7732 - mae: 0.3111 - mse: 0.1554 - auc_22: 0.7660\n",
      "Epoch 4/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4695 - accuracy: 0.7734 - mae: 0.3096 - mse: 0.1548 - auc_22: 0.7684\n",
      "Epoch 5/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4679 - accuracy: 0.7734 - mae: 0.3089 - mse: 0.1543 - auc_22: 0.7702\n",
      "Epoch 6/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4673 - accuracy: 0.7746 - mae: 0.3080 - mse: 0.1540 - auc_22: 0.7717\n",
      "Epoch 7/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4662 - accuracy: 0.7748 - mae: 0.3073 - mse: 0.1536 - auc_22: 0.7732\n",
      "Epoch 8/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4640 - accuracy: 0.7765 - mae: 0.3056 - mse: 0.1527 - auc_22: 0.7763\n",
      "Epoch 9/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4639 - accuracy: 0.7768 - mae: 0.3055 - mse: 0.1527 - auc_22: 0.7765\n",
      "Epoch 10/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4615 - accuracy: 0.7778 - mae: 0.3036 - mse: 0.1517 - auc_22: 0.7800\n",
      "Epoch 11/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4628 - accuracy: 0.7769 - mae: 0.3045 - mse: 0.1522 - auc_22: 0.7784\n",
      "Epoch 12/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4595 - accuracy: 0.7780 - mae: 0.3027 - mse: 0.1512 - auc_22: 0.7821\n",
      "Epoch 13/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4600 - accuracy: 0.7779 - mae: 0.3028 - mse: 0.1513 - auc_22: 0.7820\n",
      "Epoch 14/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4583 - accuracy: 0.7793 - mae: 0.3014 - mse: 0.1507 - auc_22: 0.7845\n",
      "Epoch 15/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4569 - accuracy: 0.7790 - mae: 0.3004 - mse: 0.1502 - auc_22: 0.7861\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "3\n",
      "781/781 [==============================] - 0s 614us/step - loss: 0.2331 - accuracy: 0.9614 - mae: 0.1917 - mse: 0.0571 - auc_22: 0.5966\n",
      "train positive label: 435 - train negative label: 224463\n",
      "up and down sampling => train positive label: 75255 - train negative label: 224463\n",
      "Test positive label: 54 - Test negative label: 24934\n",
      "Epoch 1/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5570 - accuracy: 0.7486 - mae: 0.3716 - mse: 0.1857 - auc_23: 0.5744\n",
      "Epoch 2/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5238 - accuracy: 0.7556 - mae: 0.3473 - mse: 0.1735 - auc_23: 0.6780\n",
      "Epoch 3/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5180 - accuracy: 0.7586 - mae: 0.3430 - mse: 0.1713 - auc_23: 0.6893\n",
      "Epoch 4/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5138 - accuracy: 0.7616 - mae: 0.3396 - mse: 0.1696 - auc_23: 0.6962\n",
      "Epoch 5/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5119 - accuracy: 0.7621 - mae: 0.3384 - mse: 0.1691 - auc_23: 0.6989\n",
      "Epoch 6/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5107 - accuracy: 0.7630 - mae: 0.3372 - mse: 0.1685 - auc_23: 0.7015\n",
      "Epoch 7/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5088 - accuracy: 0.7644 - mae: 0.3359 - mse: 0.1679 - auc_23: 0.7043\n",
      "Epoch 8/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5085 - accuracy: 0.7635 - mae: 0.3360 - mse: 0.1678 - auc_23: 0.7051\n",
      "Epoch 9/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5074 - accuracy: 0.7635 - mae: 0.3352 - mse: 0.1675 - auc_23: 0.7077\n",
      "Epoch 10/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5051 - accuracy: 0.7637 - mae: 0.3334 - mse: 0.1666 - auc_23: 0.7122\n",
      "Epoch 11/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5044 - accuracy: 0.7643 - mae: 0.3323 - mse: 0.1661 - auc_23: 0.7150\n",
      "Epoch 12/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5066 - accuracy: 0.7634 - mae: 0.3336 - mse: 0.1666 - auc_23: 0.7133\n",
      "Epoch 13/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5038 - accuracy: 0.7641 - mae: 0.3312 - mse: 0.1656 - auc_23: 0.7182\n",
      "Epoch 14/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5043 - accuracy: 0.7632 - mae: 0.3315 - mse: 0.1657 - auc_23: 0.7187\n",
      "Epoch 15/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5042 - accuracy: 0.7622 - mae: 0.3314 - mse: 0.1657 - auc_23: 0.7197\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "75\n",
      "74\n",
      "Epoch 1/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.5017 - accuracy: 0.7623 - mae: 0.3298 - mse: 0.1649 - auc_23: 0.7245\n",
      "Epoch 2/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4987 - accuracy: 0.7626 - mae: 0.3282 - mse: 0.1639 - auc_23: 0.7298\n",
      "Epoch 3/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4962 - accuracy: 0.7635 - mae: 0.3261 - mse: 0.1631 - auc_23: 0.7341\n",
      "Epoch 4/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4938 - accuracy: 0.7643 - mae: 0.3244 - mse: 0.1622 - auc_23: 0.7380\n",
      "Epoch 5/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4914 - accuracy: 0.7650 - mae: 0.3228 - mse: 0.1614 - auc_23: 0.7420\n",
      "Epoch 6/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4887 - accuracy: 0.7666 - mae: 0.3209 - mse: 0.1605 - auc_23: 0.7460\n",
      "Epoch 7/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4862 - accuracy: 0.7663 - mae: 0.3197 - mse: 0.1597 - auc_23: 0.7500\n",
      "Epoch 8/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4842 - accuracy: 0.7675 - mae: 0.3180 - mse: 0.1589 - auc_23: 0.7529\n",
      "Epoch 9/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4831 - accuracy: 0.7676 - mae: 0.3175 - mse: 0.1587 - auc_23: 0.7545\n",
      "Epoch 10/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4815 - accuracy: 0.7675 - mae: 0.3164 - mse: 0.1582 - auc_23: 0.7568\n",
      "Epoch 11/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4862 - accuracy: 0.7672 - mae: 0.3196 - mse: 0.1597 - auc_23: 0.7488\n",
      "Epoch 12/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4841 - accuracy: 0.7686 - mae: 0.3179 - mse: 0.1589 - auc_23: 0.7522\n",
      "Epoch 13/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4822 - accuracy: 0.7692 - mae: 0.3172 - mse: 0.1584 - auc_23: 0.7549\n",
      "Epoch 14/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4805 - accuracy: 0.7698 - mae: 0.3154 - mse: 0.1577 - auc_23: 0.7572\n",
      "Epoch 15/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4786 - accuracy: 0.7703 - mae: 0.3143 - mse: 0.1571 - auc_23: 0.7597\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "Epoch 1/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4764 - accuracy: 0.7712 - mae: 0.3131 - mse: 0.1565 - auc_23: 0.7627\n",
      "Epoch 2/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4742 - accuracy: 0.7722 - mae: 0.3117 - mse: 0.1558 - auc_23: 0.7658\n",
      "Epoch 3/15\n",
      "2342/2342 [==============================] - 9s 4ms/step - loss: 0.4737 - accuracy: 0.7719 - mae: 0.3109 - mse: 0.1554 - auc_23: 0.7667\n",
      "Epoch 4/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4727 - accuracy: 0.7735 - mae: 0.3101 - mse: 0.1550 - auc_23: 0.7681\n",
      "Epoch 5/15\n",
      "2342/2342 [==============================] - 8s 4ms/step - loss: 0.4745 - accuracy: 0.7728 - mae: 0.3116 - mse: 0.1557 - auc_23: 0.7652\n",
      "Epoch 6/15\n",
      "2342/2342 [==============================] - 9s 4ms/step - loss: 0.4709 - accuracy: 0.7737 - mae: 0.3093 - mse: 0.1546 - auc_23: 0.7699\n",
      "Epoch 7/15\n",
      "2342/2342 [==============================] - 9s 4ms/step - loss: 0.4699 - accuracy: 0.7738 - mae: 0.3088 - mse: 0.1543 - auc_23: 0.7716\n",
      "Epoch 8/15\n",
      "2342/2342 [==============================] - 9s 4ms/step - loss: 0.4671 - accuracy: 0.7763 - mae: 0.3067 - mse: 0.1533 - auc_23: 0.7749\n",
      "Epoch 9/15\n",
      "2342/2342 [==============================] - 9s 4ms/step - loss: 0.4674 - accuracy: 0.7746 - mae: 0.3073 - mse: 0.1535 - auc_23: 0.7747\n",
      "Epoch 10/15\n",
      "2342/2342 [==============================] - 9s 4ms/step - loss: 0.4646 - accuracy: 0.7763 - mae: 0.3050 - mse: 0.1525 - auc_23: 0.7785\n",
      "Epoch 11/15\n",
      "2342/2342 [==============================] - 9s 4ms/step - loss: 0.4632 - accuracy: 0.7761 - mae: 0.3044 - mse: 0.1521 - auc_23: 0.7800\n",
      "Epoch 12/15\n",
      "2342/2342 [==============================] - 9s 4ms/step - loss: 0.4625 - accuracy: 0.7759 - mae: 0.3039 - mse: 0.1519 - auc_23: 0.7812\n",
      "Epoch 13/15\n",
      "2342/2342 [==============================] - 9s 4ms/step - loss: 0.4615 - accuracy: 0.7771 - mae: 0.3034 - mse: 0.1516 - auc_23: 0.7823\n",
      "Epoch 14/15\n",
      "2342/2342 [==============================] - 9s 4ms/step - loss: 0.4602 - accuracy: 0.7770 - mae: 0.3025 - mse: 0.1513 - auc_23: 0.7834\n",
      "Epoch 15/15\n",
      "2342/2342 [==============================] - 9s 4ms/step - loss: 0.4581 - accuracy: 0.7781 - mae: 0.3013 - mse: 0.1505 - auc_23: 0.7863\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "3\n",
      "781/781 [==============================] - 0s 611us/step - loss: 0.2534 - accuracy: 0.9636 - mae: 0.2102 - mse: 0.0625 - auc_23: 0.7374\n",
      "train positive label: 437 - train negative label: 224461\n",
      "up and down sampling => train positive label: 75164 - train negative label: 224461\n",
      "Test positive label: 52 - Test negative label: 24936\n",
      "Epoch 1/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.5600 - accuracy: 0.7478 - mae: 0.3744 - mse: 0.1869 - auc_24: 0.5627\n",
      "Epoch 2/15\n",
      "2341/2341 [==============================] - 14s 6ms/step - loss: 0.5229 - accuracy: 0.7539 - mae: 0.3471 - mse: 0.1733 - auc_24: 0.6794\n",
      "Epoch 3/15\n",
      "2341/2341 [==============================] - 14s 6ms/step - loss: 0.5163 - accuracy: 0.7595 - mae: 0.3420 - mse: 0.1708 - auc_24: 0.6901\n",
      "Epoch 4/15\n",
      "2341/2341 [==============================] - 14s 6ms/step - loss: 0.5119 - accuracy: 0.7628 - mae: 0.3384 - mse: 0.1690 - auc_24: 0.6974\n",
      "Epoch 5/15\n",
      "2341/2341 [==============================] - 14s 6ms/step - loss: 0.5090 - accuracy: 0.7649 - mae: 0.3363 - mse: 0.1680 - auc_24: 0.7018\n",
      "Epoch 6/15\n",
      "2341/2341 [==============================] - 14s 6ms/step - loss: 0.5090 - accuracy: 0.7662 - mae: 0.3357 - mse: 0.1677 - auc_24: 0.7004\n",
      "Epoch 7/15\n",
      "2341/2341 [==============================] - 13s 6ms/step - loss: 0.5071 - accuracy: 0.7678 - mae: 0.3344 - mse: 0.1671 - auc_24: 0.7033\n",
      "Epoch 8/15\n",
      "2341/2341 [==============================] - 14s 6ms/step - loss: 0.5044 - accuracy: 0.7687 - mae: 0.3324 - mse: 0.1661 - auc_24: 0.7087\n",
      "Epoch 9/15\n",
      "2341/2341 [==============================] - 14s 6ms/step - loss: 0.5017 - accuracy: 0.7694 - mae: 0.3307 - mse: 0.1652 - auc_24: 0.7134\n",
      "Epoch 10/15\n",
      "2341/2341 [==============================] - 14s 6ms/step - loss: 0.5031 - accuracy: 0.7683 - mae: 0.3313 - mse: 0.1655 - auc_24: 0.7134\n",
      "Epoch 11/15\n",
      "2341/2341 [==============================] - 13s 6ms/step - loss: 0.5006 - accuracy: 0.7683 - mae: 0.3293 - mse: 0.1646 - auc_24: 0.7184\n",
      "Epoch 12/15\n",
      "2341/2341 [==============================] - 13s 6ms/step - loss: 0.4985 - accuracy: 0.7694 - mae: 0.3281 - mse: 0.1639 - auc_24: 0.7215\n",
      "Epoch 13/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4949 - accuracy: 0.7716 - mae: 0.3252 - mse: 0.1626 - auc_24: 0.7275\n",
      "Epoch 14/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4920 - accuracy: 0.7722 - mae: 0.3232 - mse: 0.1616 - auc_24: 0.7333\n",
      "Epoch 15/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4900 - accuracy: 0.7733 - mae: 0.3215 - mse: 0.1607 - auc_24: 0.7363\n",
      "77\n",
      "77\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "75\n",
      "74\n",
      "Epoch 1/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4870 - accuracy: 0.7746 - mae: 0.3196 - mse: 0.1597 - auc_24: 0.7407\n",
      "Epoch 2/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4903 - accuracy: 0.7735 - mae: 0.3216 - mse: 0.1608 - auc_24: 0.7348\n",
      "Epoch 3/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4919 - accuracy: 0.7739 - mae: 0.3227 - mse: 0.1613 - auc_24: 0.7331\n",
      "Epoch 4/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4898 - accuracy: 0.7742 - mae: 0.3213 - mse: 0.1606 - auc_24: 0.7369\n",
      "Epoch 5/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4880 - accuracy: 0.7747 - mae: 0.3200 - mse: 0.1600 - auc_24: 0.7395\n",
      "Epoch 6/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4861 - accuracy: 0.7751 - mae: 0.3189 - mse: 0.1594 - auc_24: 0.7429\n",
      "Epoch 7/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4839 - accuracy: 0.7765 - mae: 0.3173 - mse: 0.1586 - auc_24: 0.7460\n",
      "Epoch 8/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4818 - accuracy: 0.7770 - mae: 0.3159 - mse: 0.1579 - auc_24: 0.7491\n",
      "Epoch 9/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4838 - accuracy: 0.7756 - mae: 0.3175 - mse: 0.1587 - auc_24: 0.7458\n",
      "Epoch 10/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4862 - accuracy: 0.7735 - mae: 0.3194 - mse: 0.1596 - auc_24: 0.7436\n",
      "Epoch 11/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4841 - accuracy: 0.7737 - mae: 0.3181 - mse: 0.1590 - auc_24: 0.7472\n",
      "Epoch 12/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4823 - accuracy: 0.7729 - mae: 0.3170 - mse: 0.1585 - auc_24: 0.7503\n",
      "Epoch 13/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4815 - accuracy: 0.7732 - mae: 0.3165 - mse: 0.1582 - auc_24: 0.7516\n",
      "Epoch 14/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4787 - accuracy: 0.7748 - mae: 0.3146 - mse: 0.1572 - auc_24: 0.7557\n",
      "Epoch 15/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4768 - accuracy: 0.7752 - mae: 0.3133 - mse: 0.1566 - auc_24: 0.7583\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "2\n",
      "781/781 [==============================] - 1s 719us/step - loss: 0.2777 - accuracy: 0.9463 - mae: 0.2229 - mse: 0.0742 - auc_24: 0.6501\n",
      "train positive label: 447 - train negative label: 224451\n",
      "up and down sampling => train positive label: 75096 - train negative label: 224451\n",
      "Test positive label: 42 - Test negative label: 24946\n",
      "Epoch 1/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5505 - accuracy: 0.7492 - mae: 0.3669 - mse: 0.1835 - auc_25: 0.5983\n",
      "Epoch 2/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5186 - accuracy: 0.7567 - mae: 0.3440 - mse: 0.1718 - auc_25: 0.6865\n",
      "Epoch 3/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5121 - accuracy: 0.7613 - mae: 0.3392 - mse: 0.1692 - auc_25: 0.6992\n",
      "Epoch 4/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5084 - accuracy: 0.7633 - mae: 0.3359 - mse: 0.1677 - auc_25: 0.7067\n",
      "Epoch 5/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5065 - accuracy: 0.7643 - mae: 0.3342 - mse: 0.1670 - auc_25: 0.7107\n",
      "Epoch 6/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5045 - accuracy: 0.7642 - mae: 0.3325 - mse: 0.1662 - auc_25: 0.7150\n",
      "Epoch 7/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5048 - accuracy: 0.7644 - mae: 0.3329 - mse: 0.1663 - auc_25: 0.7137\n",
      "Epoch 8/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5031 - accuracy: 0.7652 - mae: 0.3318 - mse: 0.1658 - auc_25: 0.7159\n",
      "Epoch 9/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.5022 - accuracy: 0.7650 - mae: 0.3311 - mse: 0.1655 - auc_25: 0.7177\n",
      "Epoch 10/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4992 - accuracy: 0.7662 - mae: 0.3291 - mse: 0.1645 - auc_25: 0.7229\n",
      "Epoch 11/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4974 - accuracy: 0.7667 - mae: 0.3278 - mse: 0.1639 - auc_25: 0.7264\n",
      "Epoch 12/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4964 - accuracy: 0.7669 - mae: 0.3268 - mse: 0.1633 - auc_25: 0.7288\n",
      "Epoch 13/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4936 - accuracy: 0.7682 - mae: 0.3248 - mse: 0.1623 - auc_25: 0.7332\n",
      "Epoch 14/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4904 - accuracy: 0.7696 - mae: 0.3227 - mse: 0.1612 - auc_25: 0.7387\n",
      "Epoch 15/15\n",
      "2341/2341 [==============================] - 10s 4ms/step - loss: 0.4886 - accuracy: 0.7692 - mae: 0.3213 - mse: 0.1606 - auc_25: 0.7421\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "74\n",
      "Epoch 1/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4942 - accuracy: 0.7671 - mae: 0.3248 - mse: 0.1624 - auc_25: 0.7349\n",
      "Epoch 2/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4991 - accuracy: 0.7661 - mae: 0.3279 - mse: 0.1639 - auc_25: 0.7262\n",
      "Epoch 3/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4977 - accuracy: 0.7669 - mae: 0.3270 - mse: 0.1634 - auc_25: 0.7282\n",
      "Epoch 4/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4974 - accuracy: 0.7663 - mae: 0.3270 - mse: 0.1635 - auc_25: 0.7291\n",
      "Epoch 5/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4941 - accuracy: 0.7682 - mae: 0.3248 - mse: 0.1623 - auc_25: 0.7338\n",
      "Epoch 6/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4937 - accuracy: 0.7672 - mae: 0.3242 - mse: 0.1622 - auc_25: 0.7350\n",
      "Epoch 7/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4913 - accuracy: 0.7688 - mae: 0.3230 - mse: 0.1614 - auc_25: 0.7386\n",
      "Epoch 8/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4900 - accuracy: 0.7695 - mae: 0.3219 - mse: 0.1609 - auc_25: 0.7406\n",
      "Epoch 9/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4893 - accuracy: 0.7691 - mae: 0.3214 - mse: 0.1607 - auc_25: 0.7419\n",
      "Epoch 10/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4877 - accuracy: 0.7699 - mae: 0.3206 - mse: 0.1603 - auc_25: 0.7435\n",
      "Epoch 11/15\n",
      "2341/2341 [==============================] - 13s 5ms/step - loss: 0.4868 - accuracy: 0.7697 - mae: 0.3201 - mse: 0.1600 - auc_25: 0.7452\n",
      "Epoch 12/15\n",
      "2341/2341 [==============================] - 13s 6ms/step - loss: 0.4853 - accuracy: 0.7703 - mae: 0.3188 - mse: 0.1594 - auc_25: 0.7475\n",
      "Epoch 13/15\n",
      "2341/2341 [==============================] - 13s 6ms/step - loss: 0.4839 - accuracy: 0.7711 - mae: 0.3180 - mse: 0.1590 - auc_25: 0.7497\n",
      "Epoch 14/15\n",
      "2341/2341 [==============================] - 13s 6ms/step - loss: 0.4817 - accuracy: 0.7711 - mae: 0.3166 - mse: 0.1582 - auc_25: 0.7532\n",
      "Epoch 15/15\n",
      "2341/2341 [==============================] - 13s 6ms/step - loss: 0.4801 - accuracy: 0.7712 - mae: 0.3154 - mse: 0.1577 - auc_25: 0.7551\n",
      "77\n",
      "77\n",
      "77\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "Epoch 1/15\n",
      "2341/2341 [==============================] - 13s 6ms/step - loss: 0.4791 - accuracy: 0.7721 - mae: 0.3149 - mse: 0.1574 - auc_25: 0.7571\n",
      "Epoch 2/15\n",
      "2341/2341 [==============================] - 13s 6ms/step - loss: 0.4791 - accuracy: 0.7721 - mae: 0.3147 - mse: 0.1574 - auc_25: 0.7563\n",
      "Epoch 3/15\n",
      "2341/2341 [==============================] - 13s 6ms/step - loss: 0.4778 - accuracy: 0.7723 - mae: 0.3144 - mse: 0.1571 - auc_25: 0.7583\n",
      "Epoch 4/15\n",
      "2341/2341 [==============================] - 13s 5ms/step - loss: 0.4755 - accuracy: 0.7735 - mae: 0.3126 - mse: 0.1563 - auc_25: 0.7615\n",
      "Epoch 5/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4763 - accuracy: 0.7730 - mae: 0.3130 - mse: 0.1565 - auc_25: 0.7609\n",
      "Epoch 6/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4757 - accuracy: 0.7735 - mae: 0.3126 - mse: 0.1564 - auc_25: 0.7610\n",
      "Epoch 7/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4741 - accuracy: 0.7735 - mae: 0.3116 - mse: 0.1558 - auc_25: 0.7636\n",
      "Epoch 8/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4711 - accuracy: 0.7751 - mae: 0.3096 - mse: 0.1547 - auc_25: 0.7678\n",
      "Epoch 9/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4706 - accuracy: 0.7749 - mae: 0.3094 - mse: 0.1547 - auc_25: 0.7684\n",
      "Epoch 10/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4719 - accuracy: 0.7744 - mae: 0.3102 - mse: 0.1551 - auc_25: 0.7667\n",
      "Epoch 11/15\n",
      "2341/2341 [==============================] - 11s 5ms/step - loss: 0.4704 - accuracy: 0.7757 - mae: 0.3089 - mse: 0.1545 - auc_25: 0.7687\n",
      "Epoch 12/15\n",
      "2341/2341 [==============================] - 12s 5ms/step - loss: 0.4692 - accuracy: 0.7757 - mae: 0.3083 - mse: 0.1542 - auc_25: 0.7705\n",
      "Epoch 13/15\n",
      "2341/2341 [==============================] - 13s 6ms/step - loss: 0.4688 - accuracy: 0.7757 - mae: 0.3083 - mse: 0.1541 - auc_25: 0.7706\n",
      "Epoch 14/15\n",
      "2341/2341 [==============================] - 13s 6ms/step - loss: 0.4683 - accuracy: 0.7752 - mae: 0.3081 - mse: 0.1541 - auc_25: 0.7714\n",
      "Epoch 15/15\n",
      "2341/2341 [==============================] - 13s 6ms/step - loss: 0.4676 - accuracy: 0.7762 - mae: 0.3075 - mse: 0.1537 - auc_25: 0.7724\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "3\n",
      "781/781 [==============================] - 1s 968us/step - loss: 0.2205 - accuracy: 0.9705 - mae: 0.1844 - mse: 0.0523 - auc_25: 0.7068\n",
      "train positive label: 440 - train negative label: 224458\n",
      "up and down sampling => train positive label: 75240 - train negative label: 224458\n",
      "Test positive label: 49 - Test negative label: 24939\n",
      "Epoch 1/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.5575 - accuracy: 0.7476 - mae: 0.3723 - mse: 0.1860 - auc_26: 0.5784\n",
      "Epoch 2/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.5262 - accuracy: 0.7553 - mae: 0.3496 - mse: 0.1746 - auc_26: 0.6666\n",
      "Epoch 3/15\n",
      "2342/2342 [==============================] - 12s 5ms/step - loss: 0.5223 - accuracy: 0.7596 - mae: 0.3457 - mse: 0.1728 - auc_26: 0.6751\n",
      "Epoch 4/15\n",
      "2342/2342 [==============================] - 12s 5ms/step - loss: 0.5220 - accuracy: 0.7610 - mae: 0.3452 - mse: 0.1723 - auc_26: 0.6771\n",
      "Epoch 5/15\n",
      "2342/2342 [==============================] - 12s 5ms/step - loss: 0.5209 - accuracy: 0.7606 - mae: 0.3442 - mse: 0.1720 - auc_26: 0.6805\n",
      "Epoch 6/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.5191 - accuracy: 0.7609 - mae: 0.3429 - mse: 0.1713 - auc_26: 0.6845\n",
      "Epoch 7/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.5188 - accuracy: 0.7603 - mae: 0.3426 - mse: 0.1711 - auc_26: 0.6871\n",
      "Epoch 8/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.5189 - accuracy: 0.7590 - mae: 0.3426 - mse: 0.1712 - auc_26: 0.6880\n",
      "Epoch 9/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.5160 - accuracy: 0.7587 - mae: 0.3408 - mse: 0.1703 - auc_26: 0.6940\n",
      "Epoch 10/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.5127 - accuracy: 0.7584 - mae: 0.3388 - mse: 0.1693 - auc_26: 0.7006\n",
      "Epoch 11/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.5115 - accuracy: 0.7587 - mae: 0.3385 - mse: 0.1691 - auc_26: 0.7025\n",
      "Epoch 12/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.5080 - accuracy: 0.7594 - mae: 0.3360 - mse: 0.1679 - auc_26: 0.7094\n",
      "Epoch 13/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.5043 - accuracy: 0.7598 - mae: 0.3338 - mse: 0.1668 - auc_26: 0.7162\n",
      "Epoch 14/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.5017 - accuracy: 0.7596 - mae: 0.3321 - mse: 0.1659 - auc_26: 0.7212\n",
      "Epoch 15/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.5018 - accuracy: 0.7600 - mae: 0.3321 - mse: 0.1659 - auc_26: 0.7215\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "75\n",
      "74\n",
      "Epoch 1/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.5021 - accuracy: 0.7602 - mae: 0.3323 - mse: 0.1661 - auc_26: 0.7205\n",
      "Epoch 2/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.4996 - accuracy: 0.7614 - mae: 0.3305 - mse: 0.1651 - auc_26: 0.7255\n",
      "Epoch 3/15\n",
      "2342/2342 [==============================] - 12s 5ms/step - loss: 0.4956 - accuracy: 0.7629 - mae: 0.3277 - mse: 0.1637 - auc_26: 0.7319\n",
      "Epoch 4/15\n",
      "2342/2342 [==============================] - 12s 5ms/step - loss: 0.4952 - accuracy: 0.7629 - mae: 0.3272 - mse: 0.1636 - auc_26: 0.7325\n",
      "Epoch 5/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.4924 - accuracy: 0.7644 - mae: 0.3253 - mse: 0.1626 - auc_26: 0.7370\n",
      "Epoch 6/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.4909 - accuracy: 0.7645 - mae: 0.3245 - mse: 0.1621 - auc_26: 0.7396\n",
      "Epoch 7/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.4883 - accuracy: 0.7654 - mae: 0.3224 - mse: 0.1612 - auc_26: 0.7438\n",
      "Epoch 8/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.4870 - accuracy: 0.7658 - mae: 0.3218 - mse: 0.1608 - auc_26: 0.7458\n",
      "Epoch 9/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.4846 - accuracy: 0.7669 - mae: 0.3201 - mse: 0.1600 - auc_26: 0.7495\n",
      "Epoch 10/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.4829 - accuracy: 0.7682 - mae: 0.3188 - mse: 0.1593 - auc_26: 0.7517\n",
      "Epoch 11/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.4802 - accuracy: 0.7687 - mae: 0.3174 - mse: 0.1586 - auc_26: 0.7554\n",
      "Epoch 12/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.4801 - accuracy: 0.7679 - mae: 0.3171 - mse: 0.1585 - auc_26: 0.7560\n",
      "Epoch 13/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.4805 - accuracy: 0.7680 - mae: 0.3172 - mse: 0.1585 - auc_26: 0.7556\n",
      "Epoch 14/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.4774 - accuracy: 0.7686 - mae: 0.3156 - mse: 0.1578 - auc_26: 0.7594\n",
      "Epoch 15/15\n",
      "2342/2342 [==============================] - 13s 5ms/step - loss: 0.4752 - accuracy: 0.7700 - mae: 0.3137 - mse: 0.1568 - auc_26: 0.7629\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "2\n",
      "781/781 [==============================] - 1s 943us/step - loss: 0.2589 - accuracy: 0.9661 - mae: 0.2125 - mse: 0.0661 - auc_26: 0.7075\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "Epoch_S = 15\n",
    "\n",
    "def evaluate_model(df, k = 10 , shuffle = False):\n",
    "    result =[]    \n",
    "\n",
    "    kf = KFold(n_splits=10, shuffle= shuffle, random_state=None)\n",
    "    \n",
    "    for train_index, test_index in kf.split(df):\n",
    "\n",
    "        train_ds = [df[index] for index in train_index] \n",
    "        \n",
    "        valid_ds = [df[index] for index in test_index]\n",
    "        \n",
    "        label_pos , label_neg, _ , _ = count_lablel(train_ds)\n",
    "        print(f'train positive label: {label_pos} - train negative label: {label_neg}')\n",
    "        \n",
    "        train_ds = up_and_down_Samplenig(train_ds, sacale_upsampling = 2 , scale_downsampling = 0.5)\n",
    "        \n",
    "        label_pos , label_neg , _ , _ = count_lablel(train_ds)\n",
    "        print(f'up and down sampling => train positive label: {label_pos} - train negative label: {label_neg}')\n",
    "\n",
    "        label_pos , label_neg, _ , _ = count_lablel(valid_ds)\n",
    "        print(f'Test positive label: {label_pos} - Test negative label: {label_neg}')\n",
    "\n",
    "        l_train = []\n",
    "        r_train = []\n",
    "        lbls_train = []\n",
    "        l_valid = []\n",
    "        r_valid = []\n",
    "        lbls_valid = []\n",
    "\n",
    "        for i , data in enumerate(train_ds):\n",
    "            embbed_drug, onehot_task, embbed_task, lbl, task_name = data\n",
    "            l_train.append(embbed_drug[0])\n",
    "            r_train.append(embbed_task)\n",
    "            lbls_train.append(lbl.tolist())\n",
    "        \n",
    "        for i , data in enumerate(valid_ds):\n",
    "            embbed_drug, onehot_task, embbed_task, lbl, task_name = data\n",
    "            l_valid.append(embbed_drug[0])\n",
    "            r_valid.append(embbed_task)\n",
    "            lbls_valid.append(lbl.tolist())\n",
    "\n",
    "        l_train = np.array(l_train).reshape(-1,128,1)\n",
    "        r_train = np.array(r_train).reshape(-1,512,1)\n",
    "        lbls_train = np.array(lbls_train)\n",
    "\n",
    "        l_valid = np.array(l_valid).reshape(-1,128,1)\n",
    "        r_valid = np.array(r_valid).reshape(-1,512,1)\n",
    "        lbls_valid = np.array(lbls_valid)\n",
    "\n",
    "        # create neural network model\n",
    "        siamese_net = siamese_model_attentiveFp_muv()\n",
    "        history = History()\n",
    "        P = siamese_net.fit([l_train, r_train], lbls_train, epochs = Epoch_S, batch_size = 128, callbacks=[history])\n",
    "\n",
    "        for j in range(100):\n",
    "            C=1\n",
    "            Before = int(P.history['accuracy'][-1]*100)\n",
    "            for i in range(2,Epoch_S+1):\n",
    "                if  int(P.history['accuracy'][-i]*100) == Before:\n",
    "                    C=C+1\n",
    "                else:\n",
    "                    C=1\n",
    "                Before=int(P.history['accuracy'][-i]*100)\n",
    "                print(Before)\n",
    "            if C==Epoch_S:\n",
    "                break\n",
    "            P = siamese_net.fit([l_train, r_train], lbls_train, epochs = Epoch_S, batch_size = 128, callbacks=[history])\n",
    "        print(j+1)\n",
    "        \n",
    "        score  = siamese_net.evaluate([l_valid,r_valid], lbls_valid, verbose=1)\n",
    "        a = (score[1],score[4])\n",
    "        result.append(a)\n",
    "    \n",
    "    return result\n",
    "\n",
    "scores = evaluate_model(data_ds, 10, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Dropout = 0.2 and downsampling = 0.5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.8442114591598511, 0.7468533515930176),\n",
       " (0.847372829914093, 0.712647557258606),\n",
       " (0.9030373096466064, 0.63388991355896),\n",
       " (0.8979150652885437, 0.6568703651428223),\n",
       " (0.8518548011779785, 0.7641416788101196),\n",
       " (0.8734242916107178, 0.7440401911735535),\n",
       " (0.9291660189628601, 0.7043888568878174),\n",
       " (0.8433648347854614, 0.6350152492523193),\n",
       " (0.8867856860160828, 0.6838277578353882),\n",
       " (0.8933087587356567, 0.6661195755004883)]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy= 0.8770441055297852 AUC= 0.6947794497013092 STD_AUC= 0.0447423318711194\n"
     ]
    }
   ],
   "source": [
    "acc = []\n",
    "auc = []\n",
    "for i in scores:\n",
    "    acc.append(i[0])\n",
    "    auc.append(i[1])\n",
    "\n",
    "print(f'accuracy= {np.mean(acc)} AUC= {np.mean(auc)} STD_AUC= {np.std(auc)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Dropout = 0.2 and downsampling = 0.4    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.9799911975860596, 0.6839089393615723),\n",
       " (0.9724278450012207, 0.6760334372520447),\n",
       " (0.9283685088157654, 0.6512341499328613),\n",
       " (0.9422145485877991, 0.6853597164154053),\n",
       " (0.9685461521148682, 0.6880980134010315),\n",
       " (0.9613829851150513, 0.5966234803199768),\n",
       " (0.9635825157165527, 0.737411618232727),\n",
       " (0.9463342428207397, 0.6501477956771851),\n",
       " (0.9705058336257935, 0.7068324089050293),\n",
       " (0.9661437273025513, 0.7074846029281616)]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy= 0.9599497556686402 AUC= 0.6783134162425994 STD_AUC= 0.03679486363772909\n"
     ]
    }
   ],
   "source": [
    "acc = []\n",
    "auc = []\n",
    "for i in scores:\n",
    "    acc.append(i[0])\n",
    "    auc.append(i[1])\n",
    "\n",
    "print(f'accuracy= {np.mean(acc)} AUC= {np.mean(auc)} STD_AUC= {np.std(auc)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "#### Dropout = 0.3 and downsampling = 0.5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.8336067795753479, 0.7119942903518677),\n",
       " (0.8947536945343018, 0.7315952181816101),\n",
       " (0.9348112940788269, 0.6853195428848267),\n",
       " (0.9263275861740112, 0.694881796836853),\n",
       " (0.9514986872673035, 0.6474282741546631),\n",
       " (0.910320520401001, 0.6676397919654846),\n",
       " (0.8270769715309143, 0.6636998653411865),\n",
       " (0.8757003545761108, 0.6882363557815552),\n",
       " (0.9979190230369568, 0.5),\n",
       " (0.7480790615081787, 0.7459204196929932)]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy= 0.8900093972682953 AUC= 0.673671555519104\n"
     ]
    }
   ],
   "source": [
    "acc = []\n",
    "auc = []\n",
    "for i in scores:\n",
    "    acc.append(i[0])\n",
    "    auc.append(i[1])\n",
    "print(f'accuracy= {np.mean(acc)} AUC= {np.mean(auc)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "id": "_dyrk9nGcM81"
   },
   "source": [
    "## Classification with BioAct-Het and Canonical GCN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "hidden": true,
    "id": "UePCH8kkoBE5",
    "outputId": "9a7ab5d3-9594-44f5-bddb-d47a6c996228"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading GCN_canonical_MUV_pre_trained.pth from https://data.dgl.ai/dgllife/pre_trained/gcn_canonical_muv.pth...\n",
      "Pretrained model loaded\n"
     ]
    }
   ],
   "source": [
    "model_GCN = 'GCN_canonical_MUV'\n",
    "gcn_model = get_muv_model(model_GCN)\n",
    "gcn_model.eval()\n",
    "gcn_model = gcn_model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14841\n",
      "Processing molecule 2000/14841\n",
      "Processing molecule 3000/14841\n",
      "Processing molecule 4000/14841\n",
      "Processing molecule 5000/14841\n",
      "Processing molecule 6000/14841\n",
      "Processing molecule 7000/14841\n",
      "Processing molecule 8000/14841\n",
      "Processing molecule 9000/14841\n",
      "Processing molecule 10000/14841\n",
      "Processing molecule 11000/14841\n",
      "Processing molecule 12000/14841\n",
      "Processing molecule 13000/14841\n",
      "Processing molecule 14000/14841\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14734\n",
      "Processing molecule 2000/14734\n",
      "Processing molecule 3000/14734\n",
      "Processing molecule 4000/14734\n",
      "Processing molecule 5000/14734\n",
      "Processing molecule 6000/14734\n",
      "Processing molecule 7000/14734\n",
      "Processing molecule 8000/14734\n",
      "Processing molecule 9000/14734\n",
      "Processing molecule 10000/14734\n",
      "Processing molecule 11000/14734\n",
      "Processing molecule 12000/14734\n",
      "Processing molecule 13000/14734\n",
      "Processing molecule 14000/14734\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14728\n",
      "Processing molecule 2000/14728\n",
      "Processing molecule 3000/14728\n",
      "Processing molecule 4000/14728\n",
      "Processing molecule 5000/14728\n",
      "Processing molecule 6000/14728\n",
      "Processing molecule 7000/14728\n",
      "Processing molecule 8000/14728\n",
      "Processing molecule 9000/14728\n",
      "Processing molecule 10000/14728\n",
      "Processing molecule 11000/14728\n",
      "Processing molecule 12000/14728\n",
      "Processing molecule 13000/14728\n",
      "Processing molecule 14000/14728\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14623\n",
      "Processing molecule 2000/14623\n",
      "Processing molecule 3000/14623\n",
      "Processing molecule 4000/14623\n",
      "Processing molecule 5000/14623\n",
      "Processing molecule 6000/14623\n",
      "Processing molecule 7000/14623\n",
      "Processing molecule 8000/14623\n",
      "Processing molecule 9000/14623\n",
      "Processing molecule 10000/14623\n",
      "Processing molecule 11000/14623\n",
      "Processing molecule 12000/14623\n",
      "Processing molecule 13000/14623\n",
      "Processing molecule 14000/14623\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14902\n",
      "Processing molecule 2000/14902\n",
      "Processing molecule 3000/14902\n",
      "Processing molecule 4000/14902\n",
      "Processing molecule 5000/14902\n",
      "Processing molecule 6000/14902\n",
      "Processing molecule 7000/14902\n",
      "Processing molecule 8000/14902\n",
      "Processing molecule 9000/14902\n",
      "Processing molecule 10000/14902\n",
      "Processing molecule 11000/14902\n",
      "Processing molecule 12000/14902\n",
      "Processing molecule 13000/14902\n",
      "Processing molecule 14000/14902\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14601\n",
      "Processing molecule 2000/14601\n",
      "Processing molecule 3000/14601\n",
      "Processing molecule 4000/14601\n",
      "Processing molecule 5000/14601\n",
      "Processing molecule 6000/14601\n",
      "Processing molecule 7000/14601\n",
      "Processing molecule 8000/14601\n",
      "Processing molecule 9000/14601\n",
      "Processing molecule 10000/14601\n",
      "Processing molecule 11000/14601\n",
      "Processing molecule 12000/14601\n",
      "Processing molecule 13000/14601\n",
      "Processing molecule 14000/14601\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14644\n",
      "Processing molecule 2000/14644\n",
      "Processing molecule 3000/14644\n",
      "Processing molecule 4000/14644\n",
      "Processing molecule 5000/14644\n",
      "Processing molecule 6000/14644\n",
      "Processing molecule 7000/14644\n",
      "Processing molecule 8000/14644\n",
      "Processing molecule 9000/14644\n",
      "Processing molecule 10000/14644\n",
      "Processing molecule 11000/14644\n",
      "Processing molecule 12000/14644\n",
      "Processing molecule 13000/14644\n",
      "Processing molecule 14000/14644\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14411\n",
      "Processing molecule 2000/14411\n",
      "Processing molecule 3000/14411\n",
      "Processing molecule 4000/14411\n",
      "Processing molecule 5000/14411\n",
      "Processing molecule 6000/14411\n",
      "Processing molecule 7000/14411\n",
      "Processing molecule 8000/14411\n",
      "Processing molecule 9000/14411\n",
      "Processing molecule 10000/14411\n",
      "Processing molecule 11000/14411\n",
      "Processing molecule 12000/14411\n",
      "Processing molecule 13000/14411\n",
      "Processing molecule 14000/14411\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14836\n",
      "Processing molecule 2000/14836\n",
      "Processing molecule 3000/14836\n",
      "Processing molecule 4000/14836\n",
      "Processing molecule 5000/14836\n",
      "Processing molecule 6000/14836\n",
      "Processing molecule 7000/14836\n",
      "Processing molecule 8000/14836\n",
      "Processing molecule 9000/14836\n",
      "Processing molecule 10000/14836\n",
      "Processing molecule 11000/14836\n",
      "Processing molecule 12000/14836\n",
      "Processing molecule 13000/14836\n",
      "Processing molecule 14000/14836\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14682\n",
      "Processing molecule 2000/14682\n",
      "Processing molecule 3000/14682\n",
      "Processing molecule 4000/14682\n",
      "Processing molecule 5000/14682\n",
      "Processing molecule 6000/14682\n",
      "Processing molecule 7000/14682\n",
      "Processing molecule 8000/14682\n",
      "Processing molecule 9000/14682\n",
      "Processing molecule 10000/14682\n",
      "Processing molecule 11000/14682\n",
      "Processing molecule 12000/14682\n",
      "Processing molecule 13000/14682\n",
      "Processing molecule 14000/14682\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14691\n",
      "Processing molecule 2000/14691\n",
      "Processing molecule 3000/14691\n",
      "Processing molecule 4000/14691\n",
      "Processing molecule 5000/14691\n",
      "Processing molecule 6000/14691\n",
      "Processing molecule 7000/14691\n",
      "Processing molecule 8000/14691\n",
      "Processing molecule 9000/14691\n",
      "Processing molecule 10000/14691\n",
      "Processing molecule 11000/14691\n",
      "Processing molecule 12000/14691\n",
      "Processing molecule 13000/14691\n",
      "Processing molecule 14000/14691\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14644\n",
      "Processing molecule 2000/14644\n",
      "Processing molecule 3000/14644\n",
      "Processing molecule 4000/14644\n",
      "Processing molecule 5000/14644\n",
      "Processing molecule 6000/14644\n",
      "Processing molecule 7000/14644\n",
      "Processing molecule 8000/14644\n",
      "Processing molecule 9000/14644\n",
      "Processing molecule 10000/14644\n",
      "Processing molecule 11000/14644\n",
      "Processing molecule 12000/14644\n",
      "Processing molecule 13000/14644\n",
      "Processing molecule 14000/14644\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14667\n",
      "Processing molecule 2000/14667\n",
      "Processing molecule 3000/14667\n",
      "Processing molecule 4000/14667\n",
      "Processing molecule 5000/14667\n",
      "Processing molecule 6000/14667\n",
      "Processing molecule 7000/14667\n",
      "Processing molecule 8000/14667\n",
      "Processing molecule 9000/14667\n",
      "Processing molecule 10000/14667\n",
      "Processing molecule 11000/14667\n",
      "Processing molecule 12000/14667\n",
      "Processing molecule 13000/14667\n",
      "Processing molecule 14000/14667\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14711\n",
      "Processing molecule 2000/14711\n",
      "Processing molecule 3000/14711\n",
      "Processing molecule 4000/14711\n",
      "Processing molecule 5000/14711\n",
      "Processing molecule 6000/14711\n",
      "Processing molecule 7000/14711\n",
      "Processing molecule 8000/14711\n",
      "Processing molecule 9000/14711\n",
      "Processing molecule 10000/14711\n",
      "Processing molecule 11000/14711\n",
      "Processing molecule 12000/14711\n",
      "Processing molecule 13000/14711\n",
      "Processing molecule 14000/14711\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14651\n",
      "Processing molecule 2000/14651\n",
      "Processing molecule 3000/14651\n",
      "Processing molecule 4000/14651\n",
      "Processing molecule 5000/14651\n",
      "Processing molecule 6000/14651\n",
      "Processing molecule 7000/14651\n",
      "Processing molecule 8000/14651\n",
      "Processing molecule 9000/14651\n",
      "Processing molecule 10000/14651\n",
      "Processing molecule 11000/14651\n",
      "Processing molecule 12000/14651\n",
      "Processing molecule 13000/14651\n",
      "Processing molecule 14000/14651\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14774\n",
      "Processing molecule 2000/14774\n",
      "Processing molecule 3000/14774\n",
      "Processing molecule 4000/14774\n",
      "Processing molecule 5000/14774\n",
      "Processing molecule 6000/14774\n",
      "Processing molecule 7000/14774\n",
      "Processing molecule 8000/14774\n",
      "Processing molecule 9000/14774\n",
      "Processing molecule 10000/14774\n",
      "Processing molecule 11000/14774\n",
      "Processing molecule 12000/14774\n",
      "Processing molecule 13000/14774\n",
      "Processing molecule 14000/14774\n",
      "Data created!!\n",
      "Processing dgl graphs from scratch...\n",
      "Processing molecule 1000/14746\n",
      "Processing molecule 2000/14746\n",
      "Processing molecule 3000/14746\n",
      "Processing molecule 4000/14746\n",
      "Processing molecule 5000/14746\n",
      "Processing molecule 6000/14746\n",
      "Processing molecule 7000/14746\n",
      "Processing molecule 8000/14746\n",
      "Processing molecule 9000/14746\n",
      "Processing molecule 10000/14746\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing molecule 11000/14746\n",
      "Processing molecule 12000/14746\n",
      "Processing molecule 13000/14746\n",
      "Processing molecule 14000/14746\n",
      "Data created!!\n"
     ]
    }
   ],
   "source": [
    "data_ds = []\n",
    "for i, task in  enumerate(muv_tasks):\n",
    "    a = df[['smiles' , task]]\n",
    "    a = a.dropna()\n",
    "    ds = DATASET(a, smiles_to_bigraph, CanonicalAtomFeaturizer(), cache_file_path = cache_path)\n",
    "    data = create_dataset_with_gcn(ds, embed_class_muv, gcn_model, muv_tasks, i)\n",
    "    for d in data:\n",
    "        data_ds.append(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "hidden": true,
    "id": "rEpMXuj7ndA7",
    "outputId": "7d6726e0-4792-4d06-953e-648851f6117a",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train positive label: 441 - train negative label: 224456\n",
      "up and down sampling => train positive label: 112455 - train negative label: 224456\n",
      "Test positive label: 48 - Test negative label: 24941\n",
      "Epoch 1/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.6093 - accuracy: 0.6719 - mae: 0.4209 - mse: 0.2108 - auc_27: 0.6369\n",
      "Epoch 2/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5791 - accuracy: 0.6914 - mae: 0.3982 - mse: 0.1988 - auc_27: 0.6963\n",
      "Epoch 3/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5672 - accuracy: 0.6982 - mae: 0.3896 - mse: 0.1944 - auc_27: 0.7150\n",
      "Epoch 4/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5565 - accuracy: 0.7025 - mae: 0.3814 - mse: 0.1904 - auc_27: 0.7301\n",
      "Epoch 5/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5498 - accuracy: 0.7058 - mae: 0.3763 - mse: 0.1879 - auc_27: 0.7396\n",
      "Epoch 6/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5435 - accuracy: 0.7086 - mae: 0.3718 - mse: 0.1857 - auc_27: 0.7475\n",
      "Epoch 7/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5344 - accuracy: 0.7137 - mae: 0.3654 - mse: 0.1825 - auc_27: 0.7580\n",
      "Epoch 8/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5276 - accuracy: 0.7179 - mae: 0.3604 - mse: 0.1800 - auc_27: 0.7661\n",
      "Epoch 9/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5273 - accuracy: 0.7193 - mae: 0.3596 - mse: 0.1796 - auc_27: 0.7674\n",
      "Epoch 10/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5258 - accuracy: 0.7215 - mae: 0.3580 - mse: 0.1789 - auc_27: 0.7695\n",
      "Epoch 11/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5230 - accuracy: 0.7211 - mae: 0.3566 - mse: 0.1782 - auc_27: 0.7716\n",
      "Epoch 12/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5172 - accuracy: 0.7249 - mae: 0.3524 - mse: 0.1761 - auc_27: 0.7779\n",
      "Epoch 13/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5124 - accuracy: 0.7272 - mae: 0.3491 - mse: 0.1744 - auc_27: 0.7833\n",
      "Epoch 14/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5142 - accuracy: 0.7232 - mae: 0.3506 - mse: 0.1752 - auc_27: 0.7806\n",
      "Epoch 15/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5107 - accuracy: 0.7239 - mae: 0.3483 - mse: 0.1740 - auc_27: 0.7840\n",
      "72\n",
      "72\n",
      "72\n",
      "72\n",
      "72\n",
      "71\n",
      "71\n",
      "71\n",
      "70\n",
      "70\n",
      "70\n",
      "69\n",
      "69\n",
      "67\n",
      "Epoch 1/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5064 - accuracy: 0.7279 - mae: 0.3451 - mse: 0.1725 - auc_27: 0.7885\n",
      "Epoch 2/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.5006 - accuracy: 0.7312 - mae: 0.3410 - mse: 0.1704 - auc_27: 0.7944\n",
      "Epoch 3/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4977 - accuracy: 0.7329 - mae: 0.3389 - mse: 0.1693 - auc_27: 0.7975\n",
      "Epoch 4/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4968 - accuracy: 0.7336 - mae: 0.3382 - mse: 0.1690 - auc_27: 0.7982\n",
      "Epoch 5/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4939 - accuracy: 0.7353 - mae: 0.3360 - mse: 0.1679 - auc_27: 0.8013\n",
      "Epoch 6/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4904 - accuracy: 0.7381 - mae: 0.3336 - mse: 0.1666 - auc_27: 0.8046\n",
      "Epoch 7/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4891 - accuracy: 0.7388 - mae: 0.3324 - mse: 0.1661 - auc_27: 0.8062\n",
      "Epoch 8/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4849 - accuracy: 0.7411 - mae: 0.3295 - mse: 0.1646 - auc_27: 0.8100\n",
      "Epoch 9/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4831 - accuracy: 0.7424 - mae: 0.3281 - mse: 0.1639 - auc_27: 0.8117\n",
      "Epoch 10/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4798 - accuracy: 0.7441 - mae: 0.3258 - mse: 0.1628 - auc_27: 0.8147\n",
      "Epoch 11/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4776 - accuracy: 0.7440 - mae: 0.3245 - mse: 0.1622 - auc_27: 0.8161\n",
      "Epoch 12/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4767 - accuracy: 0.7461 - mae: 0.3236 - mse: 0.1617 - auc_27: 0.8172\n",
      "Epoch 13/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4744 - accuracy: 0.7470 - mae: 0.3218 - mse: 0.1608 - auc_27: 0.8196\n",
      "Epoch 14/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4730 - accuracy: 0.7489 - mae: 0.3207 - mse: 0.1603 - auc_27: 0.8210\n",
      "Epoch 15/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4710 - accuracy: 0.7492 - mae: 0.3194 - mse: 0.1596 - auc_27: 0.8224\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "72\n",
      "Epoch 1/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4686 - accuracy: 0.7506 - mae: 0.3175 - mse: 0.1587 - auc_27: 0.8248\n",
      "Epoch 2/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4666 - accuracy: 0.7527 - mae: 0.3163 - mse: 0.1580 - auc_27: 0.8265\n",
      "Epoch 3/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4647 - accuracy: 0.7531 - mae: 0.3146 - mse: 0.1573 - auc_27: 0.8282\n",
      "Epoch 4/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4643 - accuracy: 0.7543 - mae: 0.3142 - mse: 0.1570 - auc_27: 0.8290\n",
      "Epoch 5/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4619 - accuracy: 0.7553 - mae: 0.3124 - mse: 0.1561 - auc_27: 0.8310\n",
      "Epoch 6/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4598 - accuracy: 0.7574 - mae: 0.3107 - mse: 0.1553 - auc_27: 0.8330\n",
      "Epoch 7/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4594 - accuracy: 0.7568 - mae: 0.3107 - mse: 0.1553 - auc_27: 0.8330\n",
      "Epoch 8/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4587 - accuracy: 0.7573 - mae: 0.3097 - mse: 0.1548 - auc_27: 0.8339\n",
      "Epoch 9/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4555 - accuracy: 0.7592 - mae: 0.3076 - mse: 0.1537 - auc_27: 0.8365\n",
      "Epoch 10/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4558 - accuracy: 0.7595 - mae: 0.3078 - mse: 0.1538 - auc_27: 0.8365\n",
      "Epoch 11/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4528 - accuracy: 0.7608 - mae: 0.3058 - mse: 0.1528 - auc_27: 0.8387\n",
      "Epoch 12/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4535 - accuracy: 0.7596 - mae: 0.3060 - mse: 0.1530 - auc_27: 0.8383\n",
      "Epoch 13/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4516 - accuracy: 0.7614 - mae: 0.3049 - mse: 0.1524 - auc_27: 0.8397\n",
      "Epoch 14/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4502 - accuracy: 0.7615 - mae: 0.3037 - mse: 0.1518 - auc_27: 0.8408\n",
      "Epoch 15/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4491 - accuracy: 0.7629 - mae: 0.3029 - mse: 0.1514 - auc_27: 0.8419\n",
      "76\n",
      "76\n",
      "75\n",
      "76\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "Epoch 1/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4475 - accuracy: 0.7640 - mae: 0.3019 - mse: 0.1508 - auc_27: 0.8431\n",
      "Epoch 2/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4465 - accuracy: 0.7648 - mae: 0.3006 - mse: 0.1503 - auc_27: 0.8444\n",
      "Epoch 3/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4461 - accuracy: 0.7640 - mae: 0.3007 - mse: 0.1503 - auc_27: 0.8443\n",
      "Epoch 4/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4446 - accuracy: 0.7662 - mae: 0.2994 - mse: 0.1497 - auc_27: 0.8457\n",
      "Epoch 5/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4425 - accuracy: 0.7668 - mae: 0.2981 - mse: 0.1489 - auc_27: 0.8472\n",
      "Epoch 6/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4414 - accuracy: 0.7680 - mae: 0.2972 - mse: 0.1484 - auc_27: 0.8483\n",
      "Epoch 7/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4412 - accuracy: 0.7681 - mae: 0.2968 - mse: 0.1484 - auc_27: 0.8485\n",
      "Epoch 8/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4401 - accuracy: 0.7686 - mae: 0.2961 - mse: 0.1480 - auc_27: 0.8494\n",
      "Epoch 9/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4391 - accuracy: 0.7699 - mae: 0.2950 - mse: 0.1475 - auc_27: 0.8505\n",
      "Epoch 10/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4392 - accuracy: 0.7703 - mae: 0.2951 - mse: 0.1475 - auc_27: 0.8504\n",
      "Epoch 11/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4389 - accuracy: 0.7699 - mae: 0.2950 - mse: 0.1474 - auc_27: 0.8505\n",
      "Epoch 12/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4387 - accuracy: 0.7698 - mae: 0.2948 - mse: 0.1473 - auc_27: 0.8509\n",
      "Epoch 13/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4372 - accuracy: 0.7712 - mae: 0.2935 - mse: 0.1468 - auc_27: 0.8520\n",
      "Epoch 14/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4364 - accuracy: 0.7708 - mae: 0.2934 - mse: 0.1466 - auc_27: 0.8523\n",
      "Epoch 15/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4322 - accuracy: 0.7738 - mae: 0.2902 - mse: 0.1450 - auc_27: 0.8556\n",
      "77\n",
      "77\n",
      "76\n",
      "76\n",
      "77\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "Epoch 1/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4333 - accuracy: 0.7728 - mae: 0.2908 - mse: 0.1453 - auc_27: 0.8548\n",
      "Epoch 2/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4314 - accuracy: 0.7732 - mae: 0.2896 - mse: 0.1447 - auc_27: 0.8561\n",
      "Epoch 3/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4308 - accuracy: 0.7748 - mae: 0.2889 - mse: 0.1444 - auc_27: 0.8568\n",
      "Epoch 4/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4320 - accuracy: 0.7745 - mae: 0.2900 - mse: 0.1449 - auc_27: 0.8558\n",
      "Epoch 5/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4324 - accuracy: 0.7732 - mae: 0.2903 - mse: 0.1451 - auc_27: 0.8554\n",
      "Epoch 6/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4295 - accuracy: 0.7760 - mae: 0.2879 - mse: 0.1438 - auc_27: 0.8580\n",
      "Epoch 7/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4283 - accuracy: 0.7769 - mae: 0.2870 - mse: 0.1435 - auc_27: 0.8588\n",
      "Epoch 8/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4271 - accuracy: 0.7770 - mae: 0.2863 - mse: 0.1431 - auc_27: 0.8595\n",
      "Epoch 9/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4275 - accuracy: 0.7775 - mae: 0.2864 - mse: 0.1432 - auc_27: 0.8594\n",
      "Epoch 10/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4263 - accuracy: 0.7787 - mae: 0.2854 - mse: 0.1426 - auc_27: 0.8605\n",
      "Epoch 11/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4261 - accuracy: 0.7770 - mae: 0.2855 - mse: 0.1427 - auc_27: 0.8602\n",
      "Epoch 12/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4231 - accuracy: 0.7800 - mae: 0.2830 - mse: 0.1415 - auc_27: 0.8627\n",
      "Epoch 13/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4232 - accuracy: 0.7803 - mae: 0.2832 - mse: 0.1416 - auc_27: 0.8626\n",
      "Epoch 14/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4223 - accuracy: 0.7805 - mae: 0.2828 - mse: 0.1412 - auc_27: 0.8633\n",
      "Epoch 15/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4230 - accuracy: 0.7807 - mae: 0.2832 - mse: 0.1415 - auc_27: 0.8628\n",
      "78\n",
      "78\n",
      "78\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "Epoch 1/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4219 - accuracy: 0.7794 - mae: 0.2825 - mse: 0.1412 - auc_27: 0.8632\n",
      "Epoch 2/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4226 - accuracy: 0.7806 - mae: 0.2828 - mse: 0.1413 - auc_27: 0.8631\n",
      "Epoch 3/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4203 - accuracy: 0.7821 - mae: 0.2812 - mse: 0.1405 - auc_27: 0.8648\n",
      "Epoch 4/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4208 - accuracy: 0.7812 - mae: 0.2816 - mse: 0.1407 - auc_27: 0.8642\n",
      "Epoch 5/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4177 - accuracy: 0.7835 - mae: 0.2793 - mse: 0.1396 - auc_27: 0.8665\n",
      "Epoch 6/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4174 - accuracy: 0.7841 - mae: 0.2790 - mse: 0.1395 - auc_27: 0.8667\n",
      "Epoch 7/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4184 - accuracy: 0.7821 - mae: 0.2801 - mse: 0.1399 - auc_27: 0.8658\n",
      "Epoch 8/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4193 - accuracy: 0.7828 - mae: 0.2802 - mse: 0.1400 - auc_27: 0.8655\n",
      "Epoch 9/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4174 - accuracy: 0.7840 - mae: 0.2789 - mse: 0.1393 - auc_27: 0.8669\n",
      "Epoch 10/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4150 - accuracy: 0.7849 - mae: 0.2771 - mse: 0.1385 - auc_27: 0.8686\n",
      "Epoch 11/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4175 - accuracy: 0.7841 - mae: 0.2789 - mse: 0.1394 - auc_27: 0.8669\n",
      "Epoch 12/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4168 - accuracy: 0.7841 - mae: 0.2785 - mse: 0.1392 - auc_27: 0.8673\n",
      "Epoch 13/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4148 - accuracy: 0.7864 - mae: 0.2769 - mse: 0.1383 - auc_27: 0.8690\n",
      "Epoch 14/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4159 - accuracy: 0.7854 - mae: 0.2776 - mse: 0.1387 - auc_27: 0.8683\n",
      "Epoch 15/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4118 - accuracy: 0.7877 - mae: 0.2749 - mse: 0.1374 - auc_27: 0.8708\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "77\n",
      "Epoch 1/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4162 - accuracy: 0.7862 - mae: 0.2775 - mse: 0.1387 - auc_27: 0.8683\n",
      "Epoch 2/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4126 - accuracy: 0.7873 - mae: 0.2753 - mse: 0.1375 - auc_27: 0.8705\n",
      "Epoch 3/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4135 - accuracy: 0.7865 - mae: 0.2758 - mse: 0.1378 - auc_27: 0.8699\n",
      "Epoch 4/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4140 - accuracy: 0.7857 - mae: 0.2761 - mse: 0.1380 - auc_27: 0.8695\n",
      "Epoch 5/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4104 - accuracy: 0.7889 - mae: 0.2737 - mse: 0.1367 - auc_27: 0.8720\n",
      "Epoch 6/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4134 - accuracy: 0.7873 - mae: 0.2755 - mse: 0.1378 - auc_27: 0.8700\n",
      "Epoch 7/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4083 - accuracy: 0.7910 - mae: 0.2718 - mse: 0.1358 - auc_27: 0.8739\n",
      "Epoch 8/15\n",
      "2633/2633 [==============================] - 11s 4ms/step - loss: 0.4105 - accuracy: 0.7889 - mae: 0.2738 - mse: 0.1369 - auc_27: 0.8719\n",
      "Epoch 9/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4104 - accuracy: 0.7904 - mae: 0.2732 - mse: 0.1365 - auc_27: 0.8725\n",
      "Epoch 10/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4090 - accuracy: 0.7896 - mae: 0.2725 - mse: 0.1362 - auc_27: 0.8730\n",
      "Epoch 11/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4082 - accuracy: 0.7909 - mae: 0.2720 - mse: 0.1359 - auc_27: 0.8736\n",
      "Epoch 12/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4094 - accuracy: 0.7895 - mae: 0.2728 - mse: 0.1363 - auc_27: 0.8727\n",
      "Epoch 13/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4078 - accuracy: 0.7905 - mae: 0.2716 - mse: 0.1357 - auc_27: 0.8738\n",
      "Epoch 14/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4074 - accuracy: 0.7908 - mae: 0.2714 - mse: 0.1357 - auc_27: 0.8740\n",
      "Epoch 15/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4094 - accuracy: 0.7903 - mae: 0.2728 - mse: 0.1363 - auc_27: 0.8728\n",
      "79\n",
      "79\n",
      "78\n",
      "79\n",
      "78\n",
      "79\n",
      "78\n",
      "79\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "Epoch 1/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4077 - accuracy: 0.7913 - mae: 0.2717 - mse: 0.1357 - auc_27: 0.8740\n",
      "Epoch 2/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4077 - accuracy: 0.7919 - mae: 0.2715 - mse: 0.1356 - auc_27: 0.8742\n",
      "Epoch 3/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4067 - accuracy: 0.7915 - mae: 0.2704 - mse: 0.1352 - auc_27: 0.8749\n",
      "Epoch 4/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4055 - accuracy: 0.7923 - mae: 0.2701 - mse: 0.1350 - auc_27: 0.8755\n",
      "Epoch 5/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4051 - accuracy: 0.7918 - mae: 0.2696 - mse: 0.1348 - auc_27: 0.8756\n",
      "Epoch 6/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4044 - accuracy: 0.7915 - mae: 0.2695 - mse: 0.1347 - auc_27: 0.8759\n",
      "Epoch 7/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4062 - accuracy: 0.7913 - mae: 0.2703 - mse: 0.1351 - auc_27: 0.8750\n",
      "Epoch 8/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4077 - accuracy: 0.7906 - mae: 0.2715 - mse: 0.1356 - auc_27: 0.8741\n",
      "Epoch 9/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4034 - accuracy: 0.7936 - mae: 0.2682 - mse: 0.1341 - auc_27: 0.8770\n",
      "Epoch 10/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4054 - accuracy: 0.7921 - mae: 0.2696 - mse: 0.1347 - auc_27: 0.8758\n",
      "Epoch 11/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4036 - accuracy: 0.7941 - mae: 0.2682 - mse: 0.1341 - auc_27: 0.8770\n",
      "Epoch 12/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4032 - accuracy: 0.7936 - mae: 0.2678 - mse: 0.1338 - auc_27: 0.8773\n",
      "Epoch 13/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4018 - accuracy: 0.7946 - mae: 0.2672 - mse: 0.1336 - auc_27: 0.8779\n",
      "Epoch 14/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4027 - accuracy: 0.7936 - mae: 0.2677 - mse: 0.1338 - auc_27: 0.8773\n",
      "Epoch 15/15\n",
      "2633/2633 [==============================] - 10s 4ms/step - loss: 0.4034 - accuracy: 0.7937 - mae: 0.2682 - mse: 0.1340 - auc_27: 0.8771\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "8\n",
      "781/781 [==============================] - 1s 811us/step - loss: 0.3185 - accuracy: 0.8201 - mae: 0.2248 - mse: 0.1038 - auc_27: 0.7098\n",
      "train positive label: 446 - train negative label: 224451\n",
      "up and down sampling => train positive label: 112392 - train negative label: 224451\n",
      "Test positive label: 43 - Test negative label: 24946\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.6082 - accuracy: 0.6655 - mae: 0.4200 - mse: 0.2102 - auc_28: 0.6434\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5773 - accuracy: 0.6920 - mae: 0.3966 - mse: 0.1979 - auc_28: 0.7011\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5652 - accuracy: 0.6997 - mae: 0.3874 - mse: 0.1933 - auc_28: 0.7200\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5555 - accuracy: 0.7042 - mae: 0.3800 - mse: 0.1896 - auc_28: 0.7339\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5437 - accuracy: 0.7100 - mae: 0.3716 - mse: 0.1855 - auc_28: 0.7485\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5343 - accuracy: 0.7166 - mae: 0.3648 - mse: 0.1820 - auc_28: 0.7602\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5263 - accuracy: 0.7213 - mae: 0.3590 - mse: 0.1793 - auc_28: 0.7692\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5172 - accuracy: 0.7253 - mae: 0.3526 - mse: 0.1760 - auc_28: 0.7794\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5098 - accuracy: 0.7283 - mae: 0.3474 - mse: 0.1734 - auc_28: 0.7869\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5078 - accuracy: 0.7292 - mae: 0.3456 - mse: 0.1727 - auc_28: 0.7890\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5009 - accuracy: 0.7348 - mae: 0.3406 - mse: 0.1701 - auc_28: 0.7963\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4971 - accuracy: 0.7368 - mae: 0.3380 - mse: 0.1688 - auc_28: 0.7997\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4909 - accuracy: 0.7414 - mae: 0.3335 - mse: 0.1666 - auc_28: 0.8057\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4859 - accuracy: 0.7443 - mae: 0.3298 - mse: 0.1648 - auc_28: 0.8105\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4814 - accuracy: 0.7480 - mae: 0.3262 - mse: 0.1629 - auc_28: 0.8152\n",
      "74\n",
      "74\n",
      "73\n",
      "73\n",
      "72\n",
      "72\n",
      "72\n",
      "72\n",
      "71\n",
      "70\n",
      "70\n",
      "69\n",
      "69\n",
      "66\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4778 - accuracy: 0.7498 - mae: 0.3238 - mse: 0.1617 - auc_28: 0.8183\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4749 - accuracy: 0.7527 - mae: 0.3211 - mse: 0.1604 - auc_28: 0.8213\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4727 - accuracy: 0.7543 - mae: 0.3195 - mse: 0.1596 - auc_28: 0.8235\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4691 - accuracy: 0.7565 - mae: 0.3168 - mse: 0.1582 - auc_28: 0.8268\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4677 - accuracy: 0.7580 - mae: 0.3157 - mse: 0.1578 - auc_28: 0.8277\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4688 - accuracy: 0.7580 - mae: 0.3160 - mse: 0.1579 - auc_28: 0.8275\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4671 - accuracy: 0.7597 - mae: 0.3144 - mse: 0.1572 - auc_28: 0.8291\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4667 - accuracy: 0.7594 - mae: 0.3143 - mse: 0.1571 - auc_28: 0.8294\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4688 - accuracy: 0.7565 - mae: 0.3163 - mse: 0.1582 - auc_28: 0.8269\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4735 - accuracy: 0.7522 - mae: 0.3203 - mse: 0.1601 - auc_28: 0.8223\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4695 - accuracy: 0.7549 - mae: 0.3173 - mse: 0.1586 - auc_28: 0.8259\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4667 - accuracy: 0.7564 - mae: 0.3153 - mse: 0.1576 - auc_28: 0.8283\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4662 - accuracy: 0.7566 - mae: 0.3152 - mse: 0.1575 - auc_28: 0.8285\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4622 - accuracy: 0.7590 - mae: 0.3123 - mse: 0.1561 - auc_28: 0.8318\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4596 - accuracy: 0.7603 - mae: 0.3106 - mse: 0.1552 - auc_28: 0.8338\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "74\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4566 - accuracy: 0.7632 - mae: 0.3081 - mse: 0.1540 - auc_28: 0.8367\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4543 - accuracy: 0.7635 - mae: 0.3067 - mse: 0.1532 - auc_28: 0.8383\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4519 - accuracy: 0.7652 - mae: 0.3048 - mse: 0.1523 - auc_28: 0.8405\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4495 - accuracy: 0.7674 - mae: 0.3029 - mse: 0.1514 - auc_28: 0.8425\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4481 - accuracy: 0.7691 - mae: 0.3018 - mse: 0.1508 - auc_28: 0.8439\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4454 - accuracy: 0.7699 - mae: 0.2996 - mse: 0.1498 - auc_28: 0.8459\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4436 - accuracy: 0.7711 - mae: 0.2986 - mse: 0.1491 - auc_28: 0.8475\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4422 - accuracy: 0.7726 - mae: 0.2972 - mse: 0.1485 - auc_28: 0.8486\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4419 - accuracy: 0.7732 - mae: 0.2968 - mse: 0.1483 - auc_28: 0.8490\n",
      "Epoch 10/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4374 - accuracy: 0.7765 - mae: 0.2938 - mse: 0.1468 - auc_28: 0.8524\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4411 - accuracy: 0.7723 - mae: 0.2966 - mse: 0.1483 - auc_28: 0.8491\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4437 - accuracy: 0.7708 - mae: 0.2988 - mse: 0.1493 - auc_28: 0.8468\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4428 - accuracy: 0.7712 - mae: 0.2980 - mse: 0.1489 - auc_28: 0.8476\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4405 - accuracy: 0.7721 - mae: 0.2965 - mse: 0.1481 - auc_28: 0.8493\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4382 - accuracy: 0.7746 - mae: 0.2946 - mse: 0.1472 - auc_28: 0.8513\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4369 - accuracy: 0.7752 - mae: 0.2935 - mse: 0.1467 - auc_28: 0.8524\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4351 - accuracy: 0.7754 - mae: 0.2925 - mse: 0.1461 - auc_28: 0.8536\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4334 - accuracy: 0.7774 - mae: 0.2910 - mse: 0.1455 - auc_28: 0.8550\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4324 - accuracy: 0.7789 - mae: 0.2902 - mse: 0.1450 - auc_28: 0.8559\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4336 - accuracy: 0.7771 - mae: 0.2914 - mse: 0.1456 - auc_28: 0.8546\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4306 - accuracy: 0.7793 - mae: 0.2891 - mse: 0.1445 - auc_28: 0.8570\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4283 - accuracy: 0.7805 - mae: 0.2873 - mse: 0.1436 - auc_28: 0.8589\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4266 - accuracy: 0.7819 - mae: 0.2858 - mse: 0.1429 - auc_28: 0.8602\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4293 - accuracy: 0.7796 - mae: 0.2877 - mse: 0.1439 - auc_28: 0.8582\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4265 - accuracy: 0.7820 - mae: 0.2857 - mse: 0.1428 - auc_28: 0.8604\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4243 - accuracy: 0.7834 - mae: 0.2842 - mse: 0.1421 - auc_28: 0.8619\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4237 - accuracy: 0.7838 - mae: 0.2836 - mse: 0.1417 - auc_28: 0.8625\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4227 - accuracy: 0.7849 - mae: 0.2830 - mse: 0.1414 - auc_28: 0.8633\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4225 - accuracy: 0.7853 - mae: 0.2826 - mse: 0.1413 - auc_28: 0.8635\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4210 - accuracy: 0.7853 - mae: 0.2817 - mse: 0.1408 - auc_28: 0.8643\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "77\n",
      "78\n",
      "78\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4207 - accuracy: 0.7867 - mae: 0.2811 - mse: 0.1405 - auc_28: 0.8648\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4193 - accuracy: 0.7878 - mae: 0.2799 - mse: 0.1400 - auc_28: 0.8660\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4186 - accuracy: 0.7877 - mae: 0.2800 - mse: 0.1399 - auc_28: 0.8663\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 10s 4ms/step - loss: 0.4201 - accuracy: 0.7869 - mae: 0.2806 - mse: 0.1402 - auc_28: 0.8655\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4182 - accuracy: 0.7879 - mae: 0.2793 - mse: 0.1396 - auc_28: 0.8667\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4172 - accuracy: 0.7885 - mae: 0.2787 - mse: 0.1392 - auc_28: 0.8674\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4156 - accuracy: 0.7901 - mae: 0.2776 - mse: 0.1387 - auc_28: 0.8685\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4148 - accuracy: 0.7914 - mae: 0.2765 - mse: 0.1382 - auc_28: 0.8694\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4145 - accuracy: 0.7912 - mae: 0.2764 - mse: 0.1382 - auc_28: 0.8694\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4145 - accuracy: 0.7912 - mae: 0.2764 - mse: 0.1381 - auc_28: 0.8696\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4116 - accuracy: 0.7928 - mae: 0.2744 - mse: 0.1372 - auc_28: 0.8715\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4128 - accuracy: 0.7921 - mae: 0.2750 - mse: 0.1374 - auc_28: 0.8709\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4112 - accuracy: 0.7927 - mae: 0.2742 - mse: 0.1369 - auc_28: 0.8720\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4092 - accuracy: 0.7941 - mae: 0.2728 - mse: 0.1363 - auc_28: 0.8731\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4088 - accuracy: 0.7957 - mae: 0.2720 - mse: 0.1360 - auc_28: 0.8738\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4106 - accuracy: 0.7939 - mae: 0.2733 - mse: 0.1366 - auc_28: 0.8725\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4083 - accuracy: 0.7961 - mae: 0.2715 - mse: 0.1357 - auc_28: 0.8741\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4060 - accuracy: 0.7972 - mae: 0.2701 - mse: 0.1349 - auc_28: 0.8757\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4073 - accuracy: 0.7961 - mae: 0.2709 - mse: 0.1355 - auc_28: 0.8746\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4061 - accuracy: 0.7976 - mae: 0.2700 - mse: 0.1349 - auc_28: 0.8756\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4051 - accuracy: 0.7983 - mae: 0.2691 - mse: 0.1345 - auc_28: 0.8765\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4074 - accuracy: 0.7957 - mae: 0.2714 - mse: 0.1355 - auc_28: 0.8746\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4051 - accuracy: 0.7983 - mae: 0.2692 - mse: 0.1345 - auc_28: 0.8765\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4058 - accuracy: 0.7979 - mae: 0.2697 - mse: 0.1348 - auc_28: 0.8758\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4045 - accuracy: 0.7977 - mae: 0.2690 - mse: 0.1344 - auc_28: 0.8767\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4031 - accuracy: 0.7989 - mae: 0.2677 - mse: 0.1338 - auc_28: 0.8778\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4013 - accuracy: 0.8003 - mae: 0.2665 - mse: 0.1331 - auc_28: 0.8790\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4004 - accuracy: 0.8007 - mae: 0.2657 - mse: 0.1327 - auc_28: 0.8797\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4003 - accuracy: 0.8012 - mae: 0.2656 - mse: 0.1327 - auc_28: 0.8798\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4008 - accuracy: 0.8008 - mae: 0.2658 - mse: 0.1329 - auc_28: 0.8793\n",
      "80\n",
      "80\n",
      "80\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3980 - accuracy: 0.8024 - mae: 0.2640 - mse: 0.1319 - auc_28: 0.8812\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.3973 - accuracy: 0.8030 - mae: 0.2636 - mse: 0.1317 - auc_28: 0.8816\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.3962 - accuracy: 0.8031 - mae: 0.2628 - mse: 0.1313 - auc_28: 0.8822\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4007 - accuracy: 0.8012 - mae: 0.2654 - mse: 0.1327 - auc_28: 0.8797\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.3993 - accuracy: 0.8010 - mae: 0.2650 - mse: 0.1324 - auc_28: 0.8803\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.3988 - accuracy: 0.8015 - mae: 0.2648 - mse: 0.1323 - auc_28: 0.8804\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3952 - accuracy: 0.8044 - mae: 0.2621 - mse: 0.1310 - auc_28: 0.8829\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.3964 - accuracy: 0.8038 - mae: 0.2627 - mse: 0.1313 - auc_28: 0.8824\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.3963 - accuracy: 0.8031 - mae: 0.2629 - mse: 0.1314 - auc_28: 0.8822\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.3933 - accuracy: 0.8048 - mae: 0.2607 - mse: 0.1303 - auc_28: 0.8842\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.3942 - accuracy: 0.8044 - mae: 0.2614 - mse: 0.1306 - auc_28: 0.8837\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.3929 - accuracy: 0.8058 - mae: 0.2599 - mse: 0.1299 - auc_28: 0.8846\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.3929 - accuracy: 0.8063 - mae: 0.2600 - mse: 0.1299 - auc_28: 0.8847\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.3895 - accuracy: 0.8079 - mae: 0.2579 - mse: 0.1288 - auc_28: 0.8867\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.3948 - accuracy: 0.8046 - mae: 0.2613 - mse: 0.1306 - auc_28: 0.8835\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "7\n",
      "781/781 [==============================] - 1s 778us/step - loss: 0.2978 - accuracy: 0.8499 - mae: 0.2087 - mse: 0.0968 - auc_28: 0.6400\n",
      "train positive label: 439 - train negative label: 224458\n",
      "up and down sampling => train positive label: 112384 - train negative label: 224458\n",
      "Test positive label: 50 - Test negative label: 24939\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.6179 - accuracy: 0.6713 - mae: 0.4273 - mse: 0.2142 - auc_29: 0.6111\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5799 - accuracy: 0.6904 - mae: 0.3980 - mse: 0.1988 - auc_29: 0.6981\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5685 - accuracy: 0.6968 - mae: 0.3892 - mse: 0.1942 - auc_29: 0.7167\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5609 - accuracy: 0.7011 - mae: 0.3827 - mse: 0.1911 - auc_29: 0.7282\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5544 - accuracy: 0.7050 - mae: 0.3779 - mse: 0.1888 - auc_29: 0.7367\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5477 - accuracy: 0.7081 - mae: 0.3735 - mse: 0.1866 - auc_29: 0.7448\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5386 - accuracy: 0.7128 - mae: 0.3678 - mse: 0.1837 - auc_29: 0.7544\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5337 - accuracy: 0.7144 - mae: 0.3648 - mse: 0.1822 - auc_29: 0.7593\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5354 - accuracy: 0.7108 - mae: 0.3662 - mse: 0.1830 - auc_29: 0.7567\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5319 - accuracy: 0.7115 - mae: 0.3641 - mse: 0.1819 - auc_29: 0.7601\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5272 - accuracy: 0.7141 - mae: 0.3610 - mse: 0.1802 - auc_29: 0.7653\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5212 - accuracy: 0.7168 - mae: 0.3567 - mse: 0.1781 - auc_29: 0.7719\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5173 - accuracy: 0.7194 - mae: 0.3537 - mse: 0.1766 - auc_29: 0.7764\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5114 - accuracy: 0.7223 - mae: 0.3494 - mse: 0.1745 - auc_29: 0.7827\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5058 - accuracy: 0.7255 - mae: 0.3455 - mse: 0.1725 - auc_29: 0.7885\n",
      "72\n",
      "71\n",
      "71\n",
      "71\n",
      "71\n",
      "71\n",
      "71\n",
      "71\n",
      "70\n",
      "70\n",
      "70\n",
      "69\n",
      "69\n",
      "67\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5022 - accuracy: 0.7277 - mae: 0.3428 - mse: 0.1712 - auc_29: 0.7921\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4985 - accuracy: 0.7307 - mae: 0.3400 - mse: 0.1698 - auc_29: 0.7963\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4944 - accuracy: 0.7323 - mae: 0.3371 - mse: 0.1684 - auc_29: 0.8002\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4929 - accuracy: 0.7329 - mae: 0.3358 - mse: 0.1677 - auc_29: 0.8019\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4882 - accuracy: 0.7373 - mae: 0.3321 - mse: 0.1659 - auc_29: 0.8067\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4837 - accuracy: 0.7397 - mae: 0.3289 - mse: 0.1642 - auc_29: 0.8110\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4806 - accuracy: 0.7417 - mae: 0.3267 - mse: 0.1631 - auc_29: 0.8139\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4803 - accuracy: 0.7423 - mae: 0.3262 - mse: 0.1630 - auc_29: 0.8142\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4767 - accuracy: 0.7440 - mae: 0.3236 - mse: 0.1617 - auc_29: 0.8172\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4754 - accuracy: 0.7453 - mae: 0.3226 - mse: 0.1612 - auc_29: 0.8187\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4736 - accuracy: 0.7468 - mae: 0.3211 - mse: 0.1604 - auc_29: 0.8205\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4698 - accuracy: 0.7499 - mae: 0.3183 - mse: 0.1590 - auc_29: 0.8242\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4668 - accuracy: 0.7526 - mae: 0.3161 - mse: 0.1579 - auc_29: 0.8269\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4645 - accuracy: 0.7532 - mae: 0.3146 - mse: 0.1571 - auc_29: 0.8286\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4620 - accuracy: 0.7543 - mae: 0.3127 - mse: 0.1563 - auc_29: 0.8307\n",
      "75\n",
      "75\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "72\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4620 - accuracy: 0.7550 - mae: 0.3126 - mse: 0.1562 - auc_29: 0.8307\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4598 - accuracy: 0.7561 - mae: 0.3111 - mse: 0.1554 - auc_29: 0.8327\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4566 - accuracy: 0.7581 - mae: 0.3089 - mse: 0.1544 - auc_29: 0.8352\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4566 - accuracy: 0.7587 - mae: 0.3086 - mse: 0.1542 - auc_29: 0.8355\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4515 - accuracy: 0.7614 - mae: 0.3049 - mse: 0.1523 - auc_29: 0.8397\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4542 - accuracy: 0.7604 - mae: 0.3065 - mse: 0.1532 - auc_29: 0.8377\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4516 - accuracy: 0.7619 - mae: 0.3047 - mse: 0.1523 - auc_29: 0.8398\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4505 - accuracy: 0.7633 - mae: 0.3040 - mse: 0.1519 - auc_29: 0.8409\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4493 - accuracy: 0.7638 - mae: 0.3030 - mse: 0.1514 - auc_29: 0.8419\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4478 - accuracy: 0.7655 - mae: 0.3021 - mse: 0.1509 - auc_29: 0.8430\n",
      "Epoch 11/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4458 - accuracy: 0.7659 - mae: 0.3004 - mse: 0.1501 - auc_29: 0.8448\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4450 - accuracy: 0.7664 - mae: 0.2997 - mse: 0.1497 - auc_29: 0.8455\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4444 - accuracy: 0.7671 - mae: 0.2993 - mse: 0.1497 - auc_29: 0.8456\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4408 - accuracy: 0.7694 - mae: 0.2968 - mse: 0.1482 - auc_29: 0.8488\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4408 - accuracy: 0.7690 - mae: 0.2968 - mse: 0.1483 - auc_29: 0.8487\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4393 - accuracy: 0.7714 - mae: 0.2955 - mse: 0.1477 - auc_29: 0.8500\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4392 - accuracy: 0.7710 - mae: 0.2956 - mse: 0.1477 - auc_29: 0.8500\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4388 - accuracy: 0.7706 - mae: 0.2951 - mse: 0.1475 - auc_29: 0.8504\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4375 - accuracy: 0.7720 - mae: 0.2942 - mse: 0.1470 - auc_29: 0.8517\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4375 - accuracy: 0.7715 - mae: 0.2943 - mse: 0.1471 - auc_29: 0.8513\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4367 - accuracy: 0.7724 - mae: 0.2932 - mse: 0.1466 - auc_29: 0.8523\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4333 - accuracy: 0.7747 - mae: 0.2910 - mse: 0.1454 - auc_29: 0.8548\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4346 - accuracy: 0.7742 - mae: 0.2916 - mse: 0.1457 - auc_29: 0.8541\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4326 - accuracy: 0.7743 - mae: 0.2905 - mse: 0.1452 - auc_29: 0.8552\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4304 - accuracy: 0.7761 - mae: 0.2890 - mse: 0.1445 - auc_29: 0.8568\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4325 - accuracy: 0.7762 - mae: 0.2901 - mse: 0.1450 - auc_29: 0.8557\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4310 - accuracy: 0.7760 - mae: 0.2892 - mse: 0.1446 - auc_29: 0.8566\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4268 - accuracy: 0.7784 - mae: 0.2864 - mse: 0.1431 - auc_29: 0.8595\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4314 - accuracy: 0.7761 - mae: 0.2894 - mse: 0.1447 - auc_29: 0.8563\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4263 - accuracy: 0.7790 - mae: 0.2861 - mse: 0.1429 - auc_29: 0.8600\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "4\n",
      "781/781 [==============================] - 1s 742us/step - loss: 0.2981 - accuracy: 0.8862 - mae: 0.2171 - mse: 0.0948 - auc_29: 0.6834\n",
      "train positive label: 447 - train negative label: 224450\n",
      "up and down sampling => train positive label: 112644 - train negative label: 224450\n",
      "Test positive label: 42 - Test negative label: 24947\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.6086 - accuracy: 0.6696 - mae: 0.4204 - mse: 0.2105 - auc_30: 0.6403\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5776 - accuracy: 0.6954 - mae: 0.3959 - mse: 0.1976 - auc_30: 0.7019\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5676 - accuracy: 0.7039 - mae: 0.3879 - mse: 0.1935 - auc_30: 0.7185\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5604 - accuracy: 0.7075 - mae: 0.3819 - mse: 0.1905 - auc_30: 0.7301\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5504 - accuracy: 0.7125 - mae: 0.3738 - mse: 0.1866 - auc_30: 0.7435\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5450 - accuracy: 0.7177 - mae: 0.3693 - mse: 0.1844 - auc_30: 0.7499\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5402 - accuracy: 0.7233 - mae: 0.3655 - mse: 0.1826 - auc_30: 0.7554\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5352 - accuracy: 0.7269 - mae: 0.3615 - mse: 0.1807 - auc_30: 0.7620\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5294 - accuracy: 0.7301 - mae: 0.3576 - mse: 0.1786 - auc_30: 0.7687\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5223 - accuracy: 0.7337 - mae: 0.3526 - mse: 0.1762 - auc_30: 0.7765\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5159 - accuracy: 0.7364 - mae: 0.3482 - mse: 0.1740 - auc_30: 0.7835\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5132 - accuracy: 0.7377 - mae: 0.3463 - mse: 0.1731 - auc_30: 0.7861\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5101 - accuracy: 0.7376 - mae: 0.3444 - mse: 0.1720 - auc_30: 0.7896\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5161 - accuracy: 0.7313 - mae: 0.3493 - mse: 0.1745 - auc_30: 0.7824\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5180 - accuracy: 0.7281 - mae: 0.3509 - mse: 0.1753 - auc_30: 0.7804\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "72\n",
      "72\n",
      "71\n",
      "71\n",
      "70\n",
      "70\n",
      "69\n",
      "66\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5158 - accuracy: 0.7293 - mae: 0.3493 - mse: 0.1745 - auc_30: 0.7829\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5111 - accuracy: 0.7310 - mae: 0.3461 - mse: 0.1730 - auc_30: 0.7875\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5062 - accuracy: 0.7337 - mae: 0.3428 - mse: 0.1712 - auc_30: 0.7926\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5022 - accuracy: 0.7358 - mae: 0.3396 - mse: 0.1698 - auc_30: 0.7965\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5021 - accuracy: 0.7354 - mae: 0.3397 - mse: 0.1698 - auc_30: 0.7965\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4985 - accuracy: 0.7376 - mae: 0.3368 - mse: 0.1684 - auc_30: 0.8002\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4966 - accuracy: 0.7389 - mae: 0.3355 - mse: 0.1677 - auc_30: 0.8023\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4939 - accuracy: 0.7389 - mae: 0.3336 - mse: 0.1668 - auc_30: 0.8045\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4895 - accuracy: 0.7418 - mae: 0.3307 - mse: 0.1653 - auc_30: 0.8086\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4891 - accuracy: 0.7420 - mae: 0.3302 - mse: 0.1650 - auc_30: 0.8092\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4851 - accuracy: 0.7453 - mae: 0.3272 - mse: 0.1635 - auc_30: 0.8131\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4824 - accuracy: 0.7455 - mae: 0.3255 - mse: 0.1627 - auc_30: 0.8151\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4824 - accuracy: 0.7451 - mae: 0.3256 - mse: 0.1628 - auc_30: 0.8150\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4802 - accuracy: 0.7463 - mae: 0.3240 - mse: 0.1620 - auc_30: 0.8169\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4783 - accuracy: 0.7485 - mae: 0.3225 - mse: 0.1612 - auc_30: 0.8190\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "72\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4772 - accuracy: 0.7490 - mae: 0.3217 - mse: 0.1608 - auc_30: 0.8198\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4730 - accuracy: 0.7508 - mae: 0.3188 - mse: 0.1593 - auc_30: 0.8234\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4710 - accuracy: 0.7534 - mae: 0.3171 - mse: 0.1585 - auc_30: 0.8254\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4703 - accuracy: 0.7525 - mae: 0.3169 - mse: 0.1584 - auc_30: 0.8257\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4684 - accuracy: 0.7542 - mae: 0.3157 - mse: 0.1578 - auc_30: 0.8273\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4658 - accuracy: 0.7560 - mae: 0.3137 - mse: 0.1568 - auc_30: 0.8297\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4659 - accuracy: 0.7561 - mae: 0.3135 - mse: 0.1567 - auc_30: 0.8298\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4644 - accuracy: 0.7564 - mae: 0.3126 - mse: 0.1563 - auc_30: 0.8308\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4630 - accuracy: 0.7569 - mae: 0.3117 - mse: 0.1558 - auc_30: 0.8318\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4623 - accuracy: 0.7576 - mae: 0.3112 - mse: 0.1555 - auc_30: 0.8325\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4606 - accuracy: 0.7587 - mae: 0.3099 - mse: 0.1549 - auc_30: 0.8339\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4604 - accuracy: 0.7591 - mae: 0.3097 - mse: 0.1548 - auc_30: 0.8341\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4611 - accuracy: 0.7599 - mae: 0.3098 - mse: 0.1550 - auc_30: 0.8338\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4579 - accuracy: 0.7613 - mae: 0.3080 - mse: 0.1538 - auc_30: 0.8362\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4583 - accuracy: 0.7611 - mae: 0.3078 - mse: 0.1539 - auc_30: 0.8360\n",
      "76\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "74\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4566 - accuracy: 0.7617 - mae: 0.3068 - mse: 0.1533 - auc_30: 0.8374\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4545 - accuracy: 0.7631 - mae: 0.3051 - mse: 0.1524 - auc_30: 0.8393\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4507 - accuracy: 0.7654 - mae: 0.3028 - mse: 0.1513 - auc_30: 0.8419\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4555 - accuracy: 0.7629 - mae: 0.3058 - mse: 0.1529 - auc_30: 0.8385\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4512 - accuracy: 0.7658 - mae: 0.3028 - mse: 0.1514 - auc_30: 0.8418\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4504 - accuracy: 0.7654 - mae: 0.3023 - mse: 0.1510 - auc_30: 0.8424\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4496 - accuracy: 0.7664 - mae: 0.3016 - mse: 0.1508 - auc_30: 0.8431\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4561 - accuracy: 0.7611 - mae: 0.3061 - mse: 0.1530 - auc_30: 0.8379\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4514 - accuracy: 0.7654 - mae: 0.3030 - mse: 0.1515 - auc_30: 0.8415\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4468 - accuracy: 0.7678 - mae: 0.3000 - mse: 0.1499 - auc_30: 0.8450\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4478 - accuracy: 0.7672 - mae: 0.3005 - mse: 0.1503 - auc_30: 0.8440\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4459 - accuracy: 0.7682 - mae: 0.2992 - mse: 0.1495 - auc_30: 0.8457\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4446 - accuracy: 0.7690 - mae: 0.2982 - mse: 0.1491 - auc_30: 0.8466\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4439 - accuracy: 0.7699 - mae: 0.2975 - mse: 0.1488 - auc_30: 0.8473\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4437 - accuracy: 0.7711 - mae: 0.2972 - mse: 0.1485 - auc_30: 0.8477\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4422 - accuracy: 0.7706 - mae: 0.2966 - mse: 0.1483 - auc_30: 0.8483\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4440 - accuracy: 0.7699 - mae: 0.2977 - mse: 0.1488 - auc_30: 0.8471\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4417 - accuracy: 0.7716 - mae: 0.2959 - mse: 0.1479 - auc_30: 0.8491\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4398 - accuracy: 0.7730 - mae: 0.2946 - mse: 0.1472 - auc_30: 0.8506\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4411 - accuracy: 0.7726 - mae: 0.2951 - mse: 0.1475 - auc_30: 0.8499\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4387 - accuracy: 0.7738 - mae: 0.2937 - mse: 0.1468 - auc_30: 0.8514\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4420 - accuracy: 0.7711 - mae: 0.2963 - mse: 0.1481 - auc_30: 0.8486\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4388 - accuracy: 0.7742 - mae: 0.2936 - mse: 0.1468 - auc_30: 0.8516\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4378 - accuracy: 0.7748 - mae: 0.2930 - mse: 0.1464 - auc_30: 0.8522\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4367 - accuracy: 0.7757 - mae: 0.2921 - mse: 0.1460 - auc_30: 0.8531\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4346 - accuracy: 0.7763 - mae: 0.2910 - mse: 0.1454 - auc_30: 0.8541\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4338 - accuracy: 0.7767 - mae: 0.2903 - mse: 0.1451 - auc_30: 0.8551\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4338 - accuracy: 0.7774 - mae: 0.2901 - mse: 0.1450 - auc_30: 0.8551\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4357 - accuracy: 0.7765 - mae: 0.2912 - mse: 0.1457 - auc_30: 0.8539\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4300 - accuracy: 0.7799 - mae: 0.2873 - mse: 0.1436 - auc_30: 0.8579\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "76\n",
      "77\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4315 - accuracy: 0.7792 - mae: 0.2883 - mse: 0.1441 - auc_30: 0.8569\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4344 - accuracy: 0.7765 - mae: 0.2905 - mse: 0.1453 - auc_30: 0.8546\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4340 - accuracy: 0.7780 - mae: 0.2897 - mse: 0.1448 - auc_30: 0.8555\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4334 - accuracy: 0.7770 - mae: 0.2897 - mse: 0.1448 - auc_30: 0.8554\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4290 - accuracy: 0.7808 - mae: 0.2864 - mse: 0.1431 - auc_30: 0.8591\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4344 - accuracy: 0.7778 - mae: 0.2900 - mse: 0.1450 - auc_30: 0.8552\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4321 - accuracy: 0.7783 - mae: 0.2889 - mse: 0.1443 - auc_30: 0.8566\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4274 - accuracy: 0.7823 - mae: 0.2853 - mse: 0.1426 - auc_30: 0.8600\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4293 - accuracy: 0.7810 - mae: 0.2865 - mse: 0.1432 - auc_30: 0.8588\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4274 - accuracy: 0.7816 - mae: 0.2853 - mse: 0.1426 - auc_30: 0.8602\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4291 - accuracy: 0.7810 - mae: 0.2865 - mse: 0.1432 - auc_30: 0.8589\n",
      "Epoch 12/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4260 - accuracy: 0.7837 - mae: 0.2845 - mse: 0.1421 - auc_30: 0.8612\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4289 - accuracy: 0.7813 - mae: 0.2863 - mse: 0.1431 - auc_30: 0.8590\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4299 - accuracy: 0.7791 - mae: 0.2872 - mse: 0.1437 - auc_30: 0.8578\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4314 - accuracy: 0.7789 - mae: 0.2886 - mse: 0.1442 - auc_30: 0.8568\n",
      "77\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "77\n",
      "77\n",
      "78\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4279 - accuracy: 0.7808 - mae: 0.2857 - mse: 0.1428 - auc_30: 0.8596\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4305 - accuracy: 0.7796 - mae: 0.2879 - mse: 0.1438 - auc_30: 0.8575\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4265 - accuracy: 0.7828 - mae: 0.2845 - mse: 0.1423 - auc_30: 0.8608\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4228 - accuracy: 0.7854 - mae: 0.2819 - mse: 0.1409 - auc_30: 0.8632\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4215 - accuracy: 0.7863 - mae: 0.2810 - mse: 0.1405 - auc_30: 0.8641\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4266 - accuracy: 0.7825 - mae: 0.2849 - mse: 0.1424 - auc_30: 0.8603\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4271 - accuracy: 0.7825 - mae: 0.2851 - mse: 0.1425 - auc_30: 0.8600\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4222 - accuracy: 0.7854 - mae: 0.2815 - mse: 0.1407 - auc_30: 0.8637\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4221 - accuracy: 0.7859 - mae: 0.2814 - mse: 0.1406 - auc_30: 0.8638\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4268 - accuracy: 0.7825 - mae: 0.2845 - mse: 0.1423 - auc_30: 0.8607\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4209 - accuracy: 0.7869 - mae: 0.2801 - mse: 0.1401 - auc_30: 0.8649\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4210 - accuracy: 0.7867 - mae: 0.2805 - mse: 0.1402 - auc_30: 0.8647\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4205 - accuracy: 0.7870 - mae: 0.2802 - mse: 0.1401 - auc_30: 0.8651\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4193 - accuracy: 0.7879 - mae: 0.2792 - mse: 0.1396 - auc_30: 0.8661\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4204 - accuracy: 0.7872 - mae: 0.2800 - mse: 0.1399 - auc_30: 0.8652\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "77\n",
      "78\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4180 - accuracy: 0.7883 - mae: 0.2782 - mse: 0.1390 - auc_30: 0.8671\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4210 - accuracy: 0.7857 - mae: 0.2805 - mse: 0.1402 - auc_30: 0.8647\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4164 - accuracy: 0.7896 - mae: 0.2769 - mse: 0.1384 - auc_30: 0.8683\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4175 - accuracy: 0.7890 - mae: 0.2777 - mse: 0.1389 - auc_30: 0.8674\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4153 - accuracy: 0.7912 - mae: 0.2760 - mse: 0.1379 - auc_30: 0.8693\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4161 - accuracy: 0.7905 - mae: 0.2767 - mse: 0.1383 - auc_30: 0.8686\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4150 - accuracy: 0.7899 - mae: 0.2763 - mse: 0.1381 - auc_30: 0.8689\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4167 - accuracy: 0.7907 - mae: 0.2770 - mse: 0.1385 - auc_30: 0.8683\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4140 - accuracy: 0.7915 - mae: 0.2753 - mse: 0.1375 - auc_30: 0.8701\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4150 - accuracy: 0.7917 - mae: 0.2754 - mse: 0.1377 - auc_30: 0.8698\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4182 - accuracy: 0.7889 - mae: 0.2777 - mse: 0.1389 - auc_30: 0.8673\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4160 - accuracy: 0.7906 - mae: 0.2767 - mse: 0.1383 - auc_30: 0.8687\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4136 - accuracy: 0.7920 - mae: 0.2748 - mse: 0.1373 - auc_30: 0.8704\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4116 - accuracy: 0.7938 - mae: 0.2732 - mse: 0.1366 - auc_30: 0.8720\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4146 - accuracy: 0.7913 - mae: 0.2755 - mse: 0.1377 - auc_30: 0.8698\n",
      "79\n",
      "79\n",
      "79\n",
      "78\n",
      "79\n",
      "79\n",
      "79\n",
      "78\n",
      "79\n",
      "79\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4130 - accuracy: 0.7924 - mae: 0.2744 - mse: 0.1371 - auc_30: 0.8709\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4121 - accuracy: 0.7934 - mae: 0.2737 - mse: 0.1368 - auc_30: 0.8713\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4129 - accuracy: 0.7923 - mae: 0.2743 - mse: 0.1372 - auc_30: 0.8707\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4106 - accuracy: 0.7937 - mae: 0.2726 - mse: 0.1362 - auc_30: 0.8725\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4088 - accuracy: 0.7952 - mae: 0.2715 - mse: 0.1356 - auc_30: 0.8738\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4105 - accuracy: 0.7941 - mae: 0.2725 - mse: 0.1363 - auc_30: 0.8724\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4135 - accuracy: 0.7920 - mae: 0.2751 - mse: 0.1375 - auc_30: 0.8703\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4084 - accuracy: 0.7954 - mae: 0.2712 - mse: 0.1356 - auc_30: 0.8738\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4082 - accuracy: 0.7952 - mae: 0.2710 - mse: 0.1354 - auc_30: 0.8739\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4092 - accuracy: 0.7958 - mae: 0.2712 - mse: 0.1357 - auc_30: 0.8737\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4065 - accuracy: 0.7975 - mae: 0.2695 - mse: 0.1347 - auc_30: 0.8754\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4080 - accuracy: 0.7961 - mae: 0.2706 - mse: 0.1353 - auc_30: 0.8744\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4083 - accuracy: 0.7963 - mae: 0.2711 - mse: 0.1355 - auc_30: 0.8740\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4077 - accuracy: 0.7950 - mae: 0.2708 - mse: 0.1354 - auc_30: 0.8742\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4052 - accuracy: 0.7978 - mae: 0.2688 - mse: 0.1343 - auc_30: 0.8760\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "9\n",
      "781/781 [==============================] - 1s 741us/step - loss: 0.2785 - accuracy: 0.8530 - mae: 0.1966 - mse: 0.0915 - auc_30: 0.6460\n",
      "train positive label: 430 - train negative label: 224467\n",
      "up and down sampling => train positive label: 112660 - train negative label: 224467\n",
      "Test positive label: 59 - Test negative label: 24930\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.6073 - accuracy: 0.6744 - mae: 0.4192 - mse: 0.2098 - auc_31: 0.6437\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5778 - accuracy: 0.6943 - mae: 0.3972 - mse: 0.1983 - auc_31: 0.6988\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5630 - accuracy: 0.6981 - mae: 0.3864 - mse: 0.1929 - auc_31: 0.7215\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5547 - accuracy: 0.7009 - mae: 0.3801 - mse: 0.1896 - auc_31: 0.7344\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5441 - accuracy: 0.7066 - mae: 0.3721 - mse: 0.1856 - auc_31: 0.7486\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5350 - accuracy: 0.7111 - mae: 0.3652 - mse: 0.1823 - auc_31: 0.7594\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5269 - accuracy: 0.7155 - mae: 0.3594 - mse: 0.1794 - auc_31: 0.7683\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5236 - accuracy: 0.7194 - mae: 0.3570 - mse: 0.1782 - auc_31: 0.7716\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5215 - accuracy: 0.7204 - mae: 0.3556 - mse: 0.1776 - auc_31: 0.7732\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5241 - accuracy: 0.7196 - mae: 0.3572 - mse: 0.1785 - auc_31: 0.7706\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5229 - accuracy: 0.7216 - mae: 0.3564 - mse: 0.1781 - auc_31: 0.7718\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5229 - accuracy: 0.7227 - mae: 0.3560 - mse: 0.1779 - auc_31: 0.7725\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5172 - accuracy: 0.7252 - mae: 0.3521 - mse: 0.1760 - auc_31: 0.7782\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5123 - accuracy: 0.7274 - mae: 0.3488 - mse: 0.1742 - auc_31: 0.7833\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5062 - accuracy: 0.7295 - mae: 0.3443 - mse: 0.1721 - auc_31: 0.7893\n",
      "72\n",
      "72\n",
      "72\n",
      "72\n",
      "71\n",
      "72\n",
      "71\n",
      "71\n",
      "71\n",
      "70\n",
      "70\n",
      "69\n",
      "69\n",
      "67\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5060 - accuracy: 0.7287 - mae: 0.3445 - mse: 0.1721 - auc_31: 0.7890\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.5020 - accuracy: 0.7307 - mae: 0.3418 - mse: 0.1708 - auc_31: 0.7927\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4984 - accuracy: 0.7332 - mae: 0.3391 - mse: 0.1694 - auc_31: 0.7964\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4938 - accuracy: 0.7362 - mae: 0.3357 - mse: 0.1677 - auc_31: 0.8009\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4902 - accuracy: 0.7366 - mae: 0.3333 - mse: 0.1666 - auc_31: 0.8039\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4883 - accuracy: 0.7387 - mae: 0.3318 - mse: 0.1658 - auc_31: 0.8060\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4838 - accuracy: 0.7411 - mae: 0.3288 - mse: 0.1643 - auc_31: 0.8101\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4810 - accuracy: 0.7433 - mae: 0.3267 - mse: 0.1632 - auc_31: 0.8126\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4785 - accuracy: 0.7437 - mae: 0.3249 - mse: 0.1624 - auc_31: 0.8149\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4751 - accuracy: 0.7465 - mae: 0.3224 - mse: 0.1612 - auc_31: 0.8181\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4744 - accuracy: 0.7469 - mae: 0.3217 - mse: 0.1608 - auc_31: 0.8190\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4731 - accuracy: 0.7485 - mae: 0.3209 - mse: 0.1603 - auc_31: 0.8201\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4693 - accuracy: 0.7509 - mae: 0.3181 - mse: 0.1590 - auc_31: 0.8235\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4679 - accuracy: 0.7513 - mae: 0.3172 - mse: 0.1585 - auc_31: 0.8247\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4649 - accuracy: 0.7527 - mae: 0.3150 - mse: 0.1575 - auc_31: 0.8270\n",
      "75\n",
      "75\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "72\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4646 - accuracy: 0.7540 - mae: 0.3145 - mse: 0.1572 - auc_31: 0.8278\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4614 - accuracy: 0.7555 - mae: 0.3122 - mse: 0.1560 - auc_31: 0.8307\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4606 - accuracy: 0.7566 - mae: 0.3115 - mse: 0.1557 - auc_31: 0.8312\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4609 - accuracy: 0.7565 - mae: 0.3118 - mse: 0.1558 - auc_31: 0.8311\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4570 - accuracy: 0.7583 - mae: 0.3090 - mse: 0.1544 - auc_31: 0.8343\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4594 - accuracy: 0.7569 - mae: 0.3104 - mse: 0.1552 - auc_31: 0.8326\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4576 - accuracy: 0.7576 - mae: 0.3095 - mse: 0.1547 - auc_31: 0.8338\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4561 - accuracy: 0.7594 - mae: 0.3082 - mse: 0.1540 - auc_31: 0.8354\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4534 - accuracy: 0.7609 - mae: 0.3061 - mse: 0.1531 - auc_31: 0.8375\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4533 - accuracy: 0.7608 - mae: 0.3062 - mse: 0.1530 - auc_31: 0.8376\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4517 - accuracy: 0.7628 - mae: 0.3046 - mse: 0.1523 - auc_31: 0.8394\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4527 - accuracy: 0.7615 - mae: 0.3056 - mse: 0.1527 - auc_31: 0.8383\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4490 - accuracy: 0.7642 - mae: 0.3030 - mse: 0.1514 - auc_31: 0.8414\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4500 - accuracy: 0.7631 - mae: 0.3037 - mse: 0.1518 - auc_31: 0.8405\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4454 - accuracy: 0.7647 - mae: 0.3006 - mse: 0.1502 - auc_31: 0.8440\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4479 - accuracy: 0.7651 - mae: 0.3018 - mse: 0.1509 - auc_31: 0.8423\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4445 - accuracy: 0.7663 - mae: 0.2997 - mse: 0.1499 - auc_31: 0.8447\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4439 - accuracy: 0.7667 - mae: 0.2992 - mse: 0.1495 - auc_31: 0.8455\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4427 - accuracy: 0.7678 - mae: 0.2983 - mse: 0.1491 - auc_31: 0.8464\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4412 - accuracy: 0.7690 - mae: 0.2971 - mse: 0.1485 - auc_31: 0.8480\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4393 - accuracy: 0.7693 - mae: 0.2960 - mse: 0.1480 - auc_31: 0.8490\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4378 - accuracy: 0.7703 - mae: 0.2950 - mse: 0.1474 - auc_31: 0.8504\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4392 - accuracy: 0.7701 - mae: 0.2956 - mse: 0.1478 - auc_31: 0.8494\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4383 - accuracy: 0.7701 - mae: 0.2950 - mse: 0.1475 - auc_31: 0.8501\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4367 - accuracy: 0.7726 - mae: 0.2935 - mse: 0.1468 - auc_31: 0.8517\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4343 - accuracy: 0.7732 - mae: 0.2921 - mse: 0.1460 - auc_31: 0.8533\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4351 - accuracy: 0.7729 - mae: 0.2925 - mse: 0.1462 - auc_31: 0.8530\n",
      "Epoch 13/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4344 - accuracy: 0.7735 - mae: 0.2918 - mse: 0.1459 - auc_31: 0.8535\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4336 - accuracy: 0.7746 - mae: 0.2911 - mse: 0.1455 - auc_31: 0.8545\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4336 - accuracy: 0.7726 - mae: 0.2915 - mse: 0.1458 - auc_31: 0.8538\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4300 - accuracy: 0.7754 - mae: 0.2890 - mse: 0.1444 - auc_31: 0.8566\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4308 - accuracy: 0.7754 - mae: 0.2891 - mse: 0.1446 - auc_31: 0.8563\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4288 - accuracy: 0.7762 - mae: 0.2880 - mse: 0.1439 - auc_31: 0.8577\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4283 - accuracy: 0.7773 - mae: 0.2875 - mse: 0.1437 - auc_31: 0.8582\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4275 - accuracy: 0.7767 - mae: 0.2869 - mse: 0.1434 - auc_31: 0.8589\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4260 - accuracy: 0.7779 - mae: 0.2858 - mse: 0.1429 - auc_31: 0.8597\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4270 - accuracy: 0.7767 - mae: 0.2867 - mse: 0.1433 - auc_31: 0.8589\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4267 - accuracy: 0.7783 - mae: 0.2862 - mse: 0.1431 - auc_31: 0.8595\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4232 - accuracy: 0.7796 - mae: 0.2838 - mse: 0.1418 - auc_31: 0.8620\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4220 - accuracy: 0.7806 - mae: 0.2827 - mse: 0.1413 - auc_31: 0.8631\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4225 - accuracy: 0.7798 - mae: 0.2832 - mse: 0.1415 - auc_31: 0.8626\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4223 - accuracy: 0.7798 - mae: 0.2833 - mse: 0.1416 - auc_31: 0.8624\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4223 - accuracy: 0.7810 - mae: 0.2828 - mse: 0.1414 - auc_31: 0.8630\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4211 - accuracy: 0.7811 - mae: 0.2823 - mse: 0.1410 - auc_31: 0.8636\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4197 - accuracy: 0.7817 - mae: 0.2811 - mse: 0.1405 - auc_31: 0.8647\n",
      "78\n",
      "78\n",
      "77\n",
      "77\n",
      "78\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4206 - accuracy: 0.7812 - mae: 0.2819 - mse: 0.1409 - auc_31: 0.8640\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4190 - accuracy: 0.7830 - mae: 0.2802 - mse: 0.1401 - auc_31: 0.8656\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4175 - accuracy: 0.7836 - mae: 0.2795 - mse: 0.1397 - auc_31: 0.8665\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4179 - accuracy: 0.7832 - mae: 0.2799 - mse: 0.1399 - auc_31: 0.8659\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4161 - accuracy: 0.7842 - mae: 0.2783 - mse: 0.1391 - auc_31: 0.8676\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4142 - accuracy: 0.7848 - mae: 0.2770 - mse: 0.1385 - auc_31: 0.8687\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4139 - accuracy: 0.7851 - mae: 0.2769 - mse: 0.1384 - auc_31: 0.8690\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4126 - accuracy: 0.7859 - mae: 0.2758 - mse: 0.1379 - auc_31: 0.8700\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4161 - accuracy: 0.7849 - mae: 0.2784 - mse: 0.1391 - auc_31: 0.8675\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4135 - accuracy: 0.7860 - mae: 0.2764 - mse: 0.1382 - auc_31: 0.8692\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4133 - accuracy: 0.7859 - mae: 0.2766 - mse: 0.1382 - auc_31: 0.8693\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4140 - accuracy: 0.7856 - mae: 0.2767 - mse: 0.1384 - auc_31: 0.8689\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4112 - accuracy: 0.7864 - mae: 0.2749 - mse: 0.1373 - auc_31: 0.8710\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4096 - accuracy: 0.7875 - mae: 0.2736 - mse: 0.1367 - auc_31: 0.8721\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 3ms/step - loss: 0.4107 - accuracy: 0.7863 - mae: 0.2748 - mse: 0.1373 - auc_31: 0.8709\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "6\n",
      "  1/781 [..............................] - ETA: 0s - loss: 0.3213 - accuracy: 0.8125 - mae: 0.2306 - mse: 0.1110 - auc_31: 0.0000e+00WARNING:tensorflow:Callbacks method `on_test_batch_end` is slow compared to the batch time (batch time: 0.0000s vs `on_test_batch_end` time: 0.0010s). Check your callbacks.\n",
      "781/781 [==============================] - 1s 745us/step - loss: 0.2878 - accuracy: 0.8659 - mae: 0.2093 - mse: 0.0925 - auc_31: 0.7279\n",
      "train positive label: 446 - train negative label: 224451\n",
      "up and down sampling => train positive label: 112392 - train negative label: 224451\n",
      "Test positive label: 43 - Test negative label: 24946\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.6066 - accuracy: 0.6778 - mae: 0.4176 - mse: 0.2092 - auc_32: 0.6471\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5692 - accuracy: 0.7015 - mae: 0.3893 - mse: 0.1944 - auc_32: 0.7136\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5576 - accuracy: 0.7045 - mae: 0.3816 - mse: 0.1905 - auc_32: 0.7294\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5497 - accuracy: 0.7085 - mae: 0.3761 - mse: 0.1879 - auc_32: 0.7390\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5457 - accuracy: 0.7111 - mae: 0.3731 - mse: 0.1863 - auc_32: 0.7445\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5425 - accuracy: 0.7127 - mae: 0.3706 - mse: 0.1851 - auc_32: 0.7489\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5380 - accuracy: 0.7145 - mae: 0.3672 - mse: 0.1834 - auc_32: 0.7545\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5321 - accuracy: 0.7179 - mae: 0.3627 - mse: 0.1812 - auc_32: 0.7620\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 8s 3ms/step - loss: 0.5264 - accuracy: 0.7199 - mae: 0.3583 - mse: 0.1789 - auc_32: 0.7691\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 8s 3ms/step - loss: 0.5239 - accuracy: 0.7236 - mae: 0.3561 - mse: 0.1778 - auc_32: 0.7725\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 8s 3ms/step - loss: 0.5216 - accuracy: 0.7251 - mae: 0.3541 - mse: 0.1769 - auc_32: 0.7754\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 8s 3ms/step - loss: 0.5207 - accuracy: 0.7284 - mae: 0.3530 - mse: 0.1764 - auc_32: 0.7767\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 8s 3ms/step - loss: 0.5191 - accuracy: 0.7282 - mae: 0.3522 - mse: 0.1760 - auc_32: 0.7779\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 8s 3ms/step - loss: 0.5166 - accuracy: 0.7280 - mae: 0.3506 - mse: 0.1752 - auc_32: 0.7804\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 8s 3ms/step - loss: 0.5155 - accuracy: 0.7272 - mae: 0.3497 - mse: 0.1747 - auc_32: 0.7818\n",
      "72\n",
      "72\n",
      "72\n",
      "72\n",
      "72\n",
      "71\n",
      "71\n",
      "71\n",
      "71\n",
      "71\n",
      "70\n",
      "70\n",
      "70\n",
      "67\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5179 - accuracy: 0.7272 - mae: 0.3514 - mse: 0.1756 - auc_32: 0.7791\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 8s 3ms/step - loss: 0.5129 - accuracy: 0.7314 - mae: 0.3476 - mse: 0.1737 - auc_32: 0.7845\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5070 - accuracy: 0.7332 - mae: 0.3437 - mse: 0.1718 - auc_32: 0.7903\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.5011 - accuracy: 0.7381 - mae: 0.3392 - mse: 0.1696 - auc_32: 0.7965\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4974 - accuracy: 0.7408 - mae: 0.3366 - mse: 0.1682 - auc_32: 0.8001\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4909 - accuracy: 0.7446 - mae: 0.3319 - mse: 0.1659 - auc_32: 0.8064\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4879 - accuracy: 0.7457 - mae: 0.3296 - mse: 0.1648 - auc_32: 0.8090\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4845 - accuracy: 0.7490 - mae: 0.3272 - mse: 0.1634 - auc_32: 0.8126\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4801 - accuracy: 0.7513 - mae: 0.3236 - mse: 0.1618 - auc_32: 0.8169\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4780 - accuracy: 0.7526 - mae: 0.3222 - mse: 0.1610 - auc_32: 0.8188\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4748 - accuracy: 0.7540 - mae: 0.3199 - mse: 0.1599 - auc_32: 0.8215\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4705 - accuracy: 0.7569 - mae: 0.3170 - mse: 0.1585 - auc_32: 0.8251\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4671 - accuracy: 0.7596 - mae: 0.3143 - mse: 0.1571 - auc_32: 0.8281\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4664 - accuracy: 0.7595 - mae: 0.3140 - mse: 0.1569 - auc_32: 0.8288\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 10s 4ms/step - loss: 0.4617 - accuracy: 0.7620 - mae: 0.3107 - mse: 0.1552 - auc_32: 0.8326\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "73\n",
      "73\n",
      "73\n",
      "72\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4604 - accuracy: 0.7629 - mae: 0.3095 - mse: 0.1547 - auc_32: 0.8340\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4598 - accuracy: 0.7633 - mae: 0.3088 - mse: 0.1544 - auc_32: 0.8346\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4569 - accuracy: 0.7644 - mae: 0.3069 - mse: 0.1534 - auc_32: 0.8368\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4549 - accuracy: 0.7674 - mae: 0.3054 - mse: 0.1526 - auc_32: 0.8387\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4531 - accuracy: 0.7669 - mae: 0.3042 - mse: 0.1521 - auc_32: 0.8398\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4507 - accuracy: 0.7688 - mae: 0.3025 - mse: 0.1512 - auc_32: 0.8419\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4486 - accuracy: 0.7711 - mae: 0.3005 - mse: 0.1502 - auc_32: 0.8442\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4479 - accuracy: 0.7717 - mae: 0.3001 - mse: 0.1500 - auc_32: 0.8447\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4474 - accuracy: 0.7721 - mae: 0.2996 - mse: 0.1498 - auc_32: 0.8451\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4442 - accuracy: 0.7739 - mae: 0.2973 - mse: 0.1486 - auc_32: 0.8477\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4434 - accuracy: 0.7739 - mae: 0.2969 - mse: 0.1484 - auc_32: 0.8481\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4398 - accuracy: 0.7769 - mae: 0.2939 - mse: 0.1470 - auc_32: 0.8512\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4409 - accuracy: 0.7757 - mae: 0.2950 - mse: 0.1475 - auc_32: 0.8500\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4387 - accuracy: 0.7773 - mae: 0.2934 - mse: 0.1466 - auc_32: 0.8518\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4365 - accuracy: 0.7795 - mae: 0.2913 - mse: 0.1456 - auc_32: 0.8539\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4350 - accuracy: 0.7793 - mae: 0.2905 - mse: 0.1452 - auc_32: 0.8549\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4344 - accuracy: 0.7803 - mae: 0.2903 - mse: 0.1450 - auc_32: 0.8554\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4350 - accuracy: 0.7799 - mae: 0.2906 - mse: 0.1453 - auc_32: 0.8548\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4345 - accuracy: 0.7798 - mae: 0.2904 - mse: 0.1452 - auc_32: 0.8548\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4307 - accuracy: 0.7820 - mae: 0.2878 - mse: 0.1438 - auc_32: 0.8579\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4289 - accuracy: 0.7835 - mae: 0.2863 - mse: 0.1432 - auc_32: 0.8593\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4294 - accuracy: 0.7841 - mae: 0.2865 - mse: 0.1432 - auc_32: 0.8592\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4285 - accuracy: 0.7839 - mae: 0.2858 - mse: 0.1429 - auc_32: 0.8599\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4289 - accuracy: 0.7838 - mae: 0.2860 - mse: 0.1430 - auc_32: 0.8595\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4262 - accuracy: 0.7856 - mae: 0.2844 - mse: 0.1422 - auc_32: 0.8612\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4245 - accuracy: 0.7871 - mae: 0.2827 - mse: 0.1414 - auc_32: 0.8629\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4242 - accuracy: 0.7863 - mae: 0.2828 - mse: 0.1414 - auc_32: 0.8630\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4234 - accuracy: 0.7878 - mae: 0.2819 - mse: 0.1410 - auc_32: 0.8638\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4233 - accuracy: 0.7880 - mae: 0.2819 - mse: 0.1409 - auc_32: 0.8638\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4226 - accuracy: 0.7879 - mae: 0.2815 - mse: 0.1407 - auc_32: 0.8642\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "77\n",
      "77\n",
      "78\n",
      "77\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4199 - accuracy: 0.7895 - mae: 0.2798 - mse: 0.1398 - auc_32: 0.8661\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4182 - accuracy: 0.7904 - mae: 0.2785 - mse: 0.1392 - auc_32: 0.8673\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4209 - accuracy: 0.7884 - mae: 0.2803 - mse: 0.1401 - auc_32: 0.8654\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4206 - accuracy: 0.7895 - mae: 0.2798 - mse: 0.1399 - auc_32: 0.8659\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4169 - accuracy: 0.7916 - mae: 0.2774 - mse: 0.1387 - auc_32: 0.8683\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4171 - accuracy: 0.7914 - mae: 0.2775 - mse: 0.1387 - auc_32: 0.8683\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4168 - accuracy: 0.7918 - mae: 0.2772 - mse: 0.1386 - auc_32: 0.8684\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4149 - accuracy: 0.7924 - mae: 0.2758 - mse: 0.1379 - auc_32: 0.8699\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4131 - accuracy: 0.7940 - mae: 0.2747 - mse: 0.1373 - auc_32: 0.8711\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4134 - accuracy: 0.7939 - mae: 0.2745 - mse: 0.1373 - auc_32: 0.8709\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4141 - accuracy: 0.7933 - mae: 0.2751 - mse: 0.1375 - auc_32: 0.8705\n",
      "Epoch 12/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4114 - accuracy: 0.7949 - mae: 0.2734 - mse: 0.1366 - auc_32: 0.8723\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4106 - accuracy: 0.7957 - mae: 0.2727 - mse: 0.1364 - auc_32: 0.8728\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4110 - accuracy: 0.7952 - mae: 0.2731 - mse: 0.1365 - auc_32: 0.8725\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4124 - accuracy: 0.7944 - mae: 0.2739 - mse: 0.1369 - auc_32: 0.8718\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "78\n",
      "78\n",
      "79\n",
      "78\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4098 - accuracy: 0.7955 - mae: 0.2723 - mse: 0.1361 - auc_32: 0.8733\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4098 - accuracy: 0.7967 - mae: 0.2718 - mse: 0.1359 - auc_32: 0.8737\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4101 - accuracy: 0.7959 - mae: 0.2720 - mse: 0.1360 - auc_32: 0.8736\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4083 - accuracy: 0.7973 - mae: 0.2707 - mse: 0.1354 - auc_32: 0.8746\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4049 - accuracy: 0.7996 - mae: 0.2684 - mse: 0.1341 - auc_32: 0.8771\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4056 - accuracy: 0.7994 - mae: 0.2689 - mse: 0.1344 - auc_32: 0.8765\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4056 - accuracy: 0.7984 - mae: 0.2686 - mse: 0.1343 - auc_32: 0.8766\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4070 - accuracy: 0.7979 - mae: 0.2700 - mse: 0.1349 - auc_32: 0.8755\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4048 - accuracy: 0.7986 - mae: 0.2686 - mse: 0.1342 - auc_32: 0.8768\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4037 - accuracy: 0.7995 - mae: 0.2676 - mse: 0.1337 - auc_32: 0.8778\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4045 - accuracy: 0.7992 - mae: 0.2680 - mse: 0.1339 - auc_32: 0.8774\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4028 - accuracy: 0.8004 - mae: 0.2665 - mse: 0.1333 - auc_32: 0.8785\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4048 - accuracy: 0.7989 - mae: 0.2686 - mse: 0.1342 - auc_32: 0.8769\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3994 - accuracy: 0.8026 - mae: 0.2642 - mse: 0.1321 - auc_32: 0.8808\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4017 - accuracy: 0.8004 - mae: 0.2664 - mse: 0.1332 - auc_32: 0.8789\n",
      "80\n",
      "79\n",
      "80\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4012 - accuracy: 0.8008 - mae: 0.2658 - mse: 0.1329 - auc_32: 0.8792\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4021 - accuracy: 0.8008 - mae: 0.2662 - mse: 0.1331 - auc_32: 0.8789\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3992 - accuracy: 0.8017 - mae: 0.2644 - mse: 0.1321 - auc_32: 0.8809\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4027 - accuracy: 0.7999 - mae: 0.2667 - mse: 0.1334 - auc_32: 0.8784\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4010 - accuracy: 0.8018 - mae: 0.2655 - mse: 0.1327 - auc_32: 0.8797\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3984 - accuracy: 0.8036 - mae: 0.2636 - mse: 0.1317 - auc_32: 0.8815\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3968 - accuracy: 0.8036 - mae: 0.2623 - mse: 0.1312 - auc_32: 0.8825\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3994 - accuracy: 0.8026 - mae: 0.2643 - mse: 0.1321 - auc_32: 0.8808\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3953 - accuracy: 0.8049 - mae: 0.2613 - mse: 0.1306 - auc_32: 0.8835\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3968 - accuracy: 0.8045 - mae: 0.2623 - mse: 0.1311 - auc_32: 0.8826\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3956 - accuracy: 0.8045 - mae: 0.2617 - mse: 0.1308 - auc_32: 0.8832\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3944 - accuracy: 0.8071 - mae: 0.2604 - mse: 0.1302 - auc_32: 0.8843\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3935 - accuracy: 0.8060 - mae: 0.2599 - mse: 0.1299 - auc_32: 0.8847\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3957 - accuracy: 0.8045 - mae: 0.2616 - mse: 0.1308 - auc_32: 0.8832\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3959 - accuracy: 0.8044 - mae: 0.2619 - mse: 0.1309 - auc_32: 0.8830\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "79\n",
      "80\n",
      "80\n",
      "80\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3894 - accuracy: 0.8084 - mae: 0.2573 - mse: 0.1285 - auc_32: 0.8873\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3927 - accuracy: 0.8073 - mae: 0.2591 - mse: 0.1295 - auc_32: 0.8854\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3927 - accuracy: 0.8068 - mae: 0.2591 - mse: 0.1296 - auc_32: 0.8853\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3910 - accuracy: 0.8076 - mae: 0.2584 - mse: 0.1291 - auc_32: 0.8863\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3918 - accuracy: 0.8069 - mae: 0.2586 - mse: 0.1293 - auc_32: 0.8859\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3898 - accuracy: 0.8088 - mae: 0.2571 - mse: 0.1285 - auc_32: 0.8873\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3899 - accuracy: 0.8080 - mae: 0.2577 - mse: 0.1287 - auc_32: 0.8870\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3905 - accuracy: 0.8079 - mae: 0.2576 - mse: 0.1288 - auc_32: 0.8867\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3876 - accuracy: 0.8096 - mae: 0.2557 - mse: 0.1278 - auc_32: 0.8885\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3885 - accuracy: 0.8086 - mae: 0.2566 - mse: 0.1283 - auc_32: 0.8877\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3900 - accuracy: 0.8075 - mae: 0.2577 - mse: 0.1288 - auc_32: 0.8869\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3890 - accuracy: 0.8085 - mae: 0.2567 - mse: 0.1284 - auc_32: 0.8876\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3882 - accuracy: 0.8096 - mae: 0.2558 - mse: 0.1278 - auc_32: 0.8884\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3879 - accuracy: 0.8093 - mae: 0.2559 - mse: 0.1279 - auc_32: 0.8883\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.3895 - accuracy: 0.8084 - mae: 0.2570 - mse: 0.1285 - auc_32: 0.8873\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "8\n",
      "781/781 [==============================] - 0s 614us/step - loss: 0.2342 - accuracy: 0.9073 - mae: 0.1748 - mse: 0.0706 - auc_32: 0.7259\n",
      "train positive label: 439 - train negative label: 224459\n",
      "up and down sampling => train positive label: 112384 - train negative label: 224459\n",
      "Test positive label: 50 - Test negative label: 24938\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.6121 - accuracy: 0.6717 - mae: 0.4234 - mse: 0.2119 - auc_33: 0.6288\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5788 - accuracy: 0.6936 - mae: 0.3978 - mse: 0.1986 - auc_33: 0.6961\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5634 - accuracy: 0.7009 - mae: 0.3865 - mse: 0.1929 - auc_33: 0.7206\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5532 - accuracy: 0.7041 - mae: 0.3792 - mse: 0.1894 - auc_33: 0.7340\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5453 - accuracy: 0.7090 - mae: 0.3733 - mse: 0.1865 - auc_33: 0.7442\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5370 - accuracy: 0.7147 - mae: 0.3671 - mse: 0.1834 - auc_33: 0.7548\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5312 - accuracy: 0.7171 - mae: 0.3628 - mse: 0.1813 - auc_33: 0.7617\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5275 - accuracy: 0.7200 - mae: 0.3603 - mse: 0.1799 - auc_33: 0.7658\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5225 - accuracy: 0.7234 - mae: 0.3565 - mse: 0.1781 - auc_33: 0.7717\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5165 - accuracy: 0.7276 - mae: 0.3522 - mse: 0.1759 - auc_33: 0.7783\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5131 - accuracy: 0.7308 - mae: 0.3494 - mse: 0.1746 - auc_33: 0.7821\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5170 - accuracy: 0.7291 - mae: 0.3525 - mse: 0.1761 - auc_33: 0.7767\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5128 - accuracy: 0.7319 - mae: 0.3495 - mse: 0.1746 - auc_33: 0.7813\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5070 - accuracy: 0.7346 - mae: 0.3454 - mse: 0.1726 - auc_33: 0.7875\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5029 - accuracy: 0.7368 - mae: 0.3427 - mse: 0.1712 - auc_33: 0.7916\n",
      "73\n",
      "73\n",
      "72\n",
      "73\n",
      "72\n",
      "72\n",
      "72\n",
      "71\n",
      "71\n",
      "70\n",
      "70\n",
      "70\n",
      "69\n",
      "67\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.5001 - accuracy: 0.7396 - mae: 0.3402 - mse: 0.1700 - auc_33: 0.7951\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4969 - accuracy: 0.7431 - mae: 0.3375 - mse: 0.1687 - auc_33: 0.7990\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4902 - accuracy: 0.7471 - mae: 0.3329 - mse: 0.1663 - auc_33: 0.8053\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4896 - accuracy: 0.7461 - mae: 0.3325 - mse: 0.1662 - auc_33: 0.8058\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4938 - accuracy: 0.7438 - mae: 0.3354 - mse: 0.1676 - auc_33: 0.8015\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4910 - accuracy: 0.7458 - mae: 0.3333 - mse: 0.1666 - auc_33: 0.8043\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4861 - accuracy: 0.7481 - mae: 0.3300 - mse: 0.1649 - auc_33: 0.8089\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4821 - accuracy: 0.7510 - mae: 0.3268 - mse: 0.1633 - auc_33: 0.8132\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4793 - accuracy: 0.7519 - mae: 0.3249 - mse: 0.1623 - auc_33: 0.8157\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4777 - accuracy: 0.7525 - mae: 0.3237 - mse: 0.1618 - auc_33: 0.8172\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4728 - accuracy: 0.7555 - mae: 0.3202 - mse: 0.1600 - auc_33: 0.8217\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4715 - accuracy: 0.7562 - mae: 0.3191 - mse: 0.1594 - auc_33: 0.8231\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4686 - accuracy: 0.7573 - mae: 0.3171 - mse: 0.1585 - auc_33: 0.8256\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4660 - accuracy: 0.7591 - mae: 0.3151 - mse: 0.1575 - auc_33: 0.8280\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4644 - accuracy: 0.7596 - mae: 0.3141 - mse: 0.1569 - auc_33: 0.8293\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "73\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4629 - accuracy: 0.7604 - mae: 0.3130 - mse: 0.1564 - auc_33: 0.8305\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4607 - accuracy: 0.7626 - mae: 0.3109 - mse: 0.1553 - auc_33: 0.8330\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4599 - accuracy: 0.7626 - mae: 0.3105 - mse: 0.1551 - auc_33: 0.8335\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4565 - accuracy: 0.7651 - mae: 0.3082 - mse: 0.1539 - auc_33: 0.8364\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4531 - accuracy: 0.7670 - mae: 0.3056 - mse: 0.1527 - auc_33: 0.8393\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4541 - accuracy: 0.7659 - mae: 0.3063 - mse: 0.1530 - auc_33: 0.8385\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4512 - accuracy: 0.7668 - mae: 0.3045 - mse: 0.1520 - auc_33: 0.8407\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4500 - accuracy: 0.7687 - mae: 0.3031 - mse: 0.1514 - auc_33: 0.8421\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4480 - accuracy: 0.7690 - mae: 0.3021 - mse: 0.1509 - auc_33: 0.8433\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4458 - accuracy: 0.7710 - mae: 0.3003 - mse: 0.1501 - auc_33: 0.8452\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4458 - accuracy: 0.7712 - mae: 0.3000 - mse: 0.1498 - auc_33: 0.8456\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4439 - accuracy: 0.7728 - mae: 0.2986 - mse: 0.1492 - auc_33: 0.8471\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4432 - accuracy: 0.7725 - mae: 0.2981 - mse: 0.1489 - auc_33: 0.8478\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4414 - accuracy: 0.7747 - mae: 0.2966 - mse: 0.1482 - auc_33: 0.8492\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4394 - accuracy: 0.7746 - mae: 0.2955 - mse: 0.1476 - auc_33: 0.8507\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4382 - accuracy: 0.7761 - mae: 0.2943 - mse: 0.1471 - auc_33: 0.8517\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4384 - accuracy: 0.7763 - mae: 0.2942 - mse: 0.1470 - auc_33: 0.8518\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4393 - accuracy: 0.7763 - mae: 0.2951 - mse: 0.1474 - auc_33: 0.8511\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4375 - accuracy: 0.7774 - mae: 0.2935 - mse: 0.1466 - auc_33: 0.8527\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4372 - accuracy: 0.7768 - mae: 0.2937 - mse: 0.1467 - auc_33: 0.8526\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4344 - accuracy: 0.7793 - mae: 0.2915 - mse: 0.1456 - auc_33: 0.8548\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4327 - accuracy: 0.7801 - mae: 0.2901 - mse: 0.1450 - auc_33: 0.8562\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4330 - accuracy: 0.7789 - mae: 0.2905 - mse: 0.1452 - auc_33: 0.8558\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4312 - accuracy: 0.7811 - mae: 0.2890 - mse: 0.1443 - auc_33: 0.8575\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4304 - accuracy: 0.7806 - mae: 0.2888 - mse: 0.1443 - auc_33: 0.8577\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4307 - accuracy: 0.7807 - mae: 0.2888 - mse: 0.1442 - auc_33: 0.8577\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4273 - accuracy: 0.7830 - mae: 0.2862 - mse: 0.1429 - auc_33: 0.8604\n",
      "Epoch 13/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4282 - accuracy: 0.7822 - mae: 0.2869 - mse: 0.1433 - auc_33: 0.8596\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4255 - accuracy: 0.7843 - mae: 0.2850 - mse: 0.1423 - auc_33: 0.8616\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4258 - accuracy: 0.7838 - mae: 0.2852 - mse: 0.1425 - auc_33: 0.8612\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "77\n",
      "78\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4246 - accuracy: 0.7845 - mae: 0.2843 - mse: 0.1420 - auc_33: 0.8622\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4236 - accuracy: 0.7865 - mae: 0.2831 - mse: 0.1413 - auc_33: 0.8635\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4247 - accuracy: 0.7856 - mae: 0.2840 - mse: 0.1418 - auc_33: 0.8627\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4232 - accuracy: 0.7854 - mae: 0.2829 - mse: 0.1413 - auc_33: 0.8636\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 3ms/step - loss: 0.4210 - accuracy: 0.7871 - mae: 0.2816 - mse: 0.1406 - auc_33: 0.8651\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4206 - accuracy: 0.7872 - mae: 0.2813 - mse: 0.1405 - auc_33: 0.8653\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4190 - accuracy: 0.7882 - mae: 0.2803 - mse: 0.1400 - auc_33: 0.8662\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4210 - accuracy: 0.7869 - mae: 0.2816 - mse: 0.1407 - auc_33: 0.8650\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4174 - accuracy: 0.7892 - mae: 0.2792 - mse: 0.1393 - auc_33: 0.8675\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4163 - accuracy: 0.7904 - mae: 0.2781 - mse: 0.1389 - auc_33: 0.8684\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4167 - accuracy: 0.7898 - mae: 0.2783 - mse: 0.1390 - auc_33: 0.8681\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4166 - accuracy: 0.7899 - mae: 0.2783 - mse: 0.1390 - auc_33: 0.8683\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4149 - accuracy: 0.7907 - mae: 0.2770 - mse: 0.1384 - auc_33: 0.8693\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4151 - accuracy: 0.7911 - mae: 0.2771 - mse: 0.1383 - auc_33: 0.8695\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4152 - accuracy: 0.7900 - mae: 0.2776 - mse: 0.1386 - auc_33: 0.8689\n",
      "79\n",
      "79\n",
      "78\n",
      "78\n",
      "79\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "Epoch 1/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4141 - accuracy: 0.7913 - mae: 0.2765 - mse: 0.1381 - auc_33: 0.8699\n",
      "Epoch 2/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4108 - accuracy: 0.7939 - mae: 0.2737 - mse: 0.1366 - auc_33: 0.8727\n",
      "Epoch 3/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4136 - accuracy: 0.7917 - mae: 0.2760 - mse: 0.1378 - auc_33: 0.8704\n",
      "Epoch 4/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4098 - accuracy: 0.7953 - mae: 0.2731 - mse: 0.1363 - auc_33: 0.8733\n",
      "Epoch 5/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4106 - accuracy: 0.7937 - mae: 0.2739 - mse: 0.1368 - auc_33: 0.8724\n",
      "Epoch 6/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4097 - accuracy: 0.7946 - mae: 0.2730 - mse: 0.1363 - auc_33: 0.8734\n",
      "Epoch 7/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4079 - accuracy: 0.7961 - mae: 0.2716 - mse: 0.1357 - auc_33: 0.8745\n",
      "Epoch 8/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4079 - accuracy: 0.7960 - mae: 0.2717 - mse: 0.1356 - auc_33: 0.8747\n",
      "Epoch 9/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4075 - accuracy: 0.7964 - mae: 0.2713 - mse: 0.1354 - auc_33: 0.8749\n",
      "Epoch 10/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4076 - accuracy: 0.7965 - mae: 0.2714 - mse: 0.1355 - auc_33: 0.8747\n",
      "Epoch 11/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4046 - accuracy: 0.7979 - mae: 0.2694 - mse: 0.1344 - auc_33: 0.8767\n",
      "Epoch 12/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4060 - accuracy: 0.7967 - mae: 0.2702 - mse: 0.1350 - auc_33: 0.8758\n",
      "Epoch 13/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4043 - accuracy: 0.7984 - mae: 0.2691 - mse: 0.1343 - auc_33: 0.8769\n",
      "Epoch 14/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4049 - accuracy: 0.7975 - mae: 0.2694 - mse: 0.1346 - auc_33: 0.8764\n",
      "Epoch 15/15\n",
      "2632/2632 [==============================] - 9s 4ms/step - loss: 0.4047 - accuracy: 0.7982 - mae: 0.2692 - mse: 0.1344 - auc_33: 0.8768\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "6\n",
      "781/781 [==============================] - 1s 652us/step - loss: 0.2922 - accuracy: 0.8746 - mae: 0.2076 - mse: 0.0933 - auc_33: 0.6932\n",
      "train positive label: 438 - train negative label: 224460\n",
      "up and down sampling => train positive label: 112566 - train negative label: 224460\n",
      "Test positive label: 51 - Test negative label: 24937\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.6103 - accuracy: 0.6704 - mae: 0.4216 - mse: 0.2110 - auc_34: 0.6409\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5816 - accuracy: 0.6940 - mae: 0.3992 - mse: 0.1992 - auc_34: 0.6933\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5715 - accuracy: 0.6998 - mae: 0.3910 - mse: 0.1952 - auc_34: 0.7113\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5639 - accuracy: 0.7025 - mae: 0.3856 - mse: 0.1924 - auc_34: 0.7226\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5552 - accuracy: 0.7081 - mae: 0.3789 - mse: 0.1890 - auc_34: 0.7353\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5476 - accuracy: 0.7121 - mae: 0.3732 - mse: 0.1862 - auc_34: 0.7450\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5444 - accuracy: 0.7128 - mae: 0.3708 - mse: 0.1852 - auc_34: 0.7486\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5375 - accuracy: 0.7164 - mae: 0.3661 - mse: 0.1828 - auc_34: 0.7565\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5310 - accuracy: 0.7200 - mae: 0.3612 - mse: 0.1803 - auc_34: 0.7640\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5254 - accuracy: 0.7226 - mae: 0.3572 - mse: 0.1784 - auc_34: 0.7708\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5293 - accuracy: 0.7212 - mae: 0.3593 - mse: 0.1796 - auc_34: 0.7664\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5323 - accuracy: 0.7203 - mae: 0.3617 - mse: 0.1808 - auc_34: 0.7629\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5283 - accuracy: 0.7210 - mae: 0.3589 - mse: 0.1795 - auc_34: 0.7669\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5221 - accuracy: 0.7247 - mae: 0.3549 - mse: 0.1773 - auc_34: 0.7740\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5178 - accuracy: 0.7261 - mae: 0.3517 - mse: 0.1757 - auc_34: 0.7784\n",
      "72\n",
      "72\n",
      "72\n",
      "72\n",
      "72\n",
      "72\n",
      "71\n",
      "71\n",
      "71\n",
      "70\n",
      "70\n",
      "69\n",
      "69\n",
      "67\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5135 - accuracy: 0.7288 - mae: 0.3487 - mse: 0.1742 - auc_34: 0.7831\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5101 - accuracy: 0.7295 - mae: 0.3465 - mse: 0.1731 - auc_34: 0.7861\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5061 - accuracy: 0.7322 - mae: 0.3435 - mse: 0.1716 - auc_34: 0.7906\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5027 - accuracy: 0.7338 - mae: 0.3413 - mse: 0.1704 - auc_34: 0.7940\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4982 - accuracy: 0.7360 - mae: 0.3381 - mse: 0.1689 - auc_34: 0.7980\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4988 - accuracy: 0.7359 - mae: 0.3385 - mse: 0.1690 - auc_34: 0.7979\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4951 - accuracy: 0.7376 - mae: 0.3358 - mse: 0.1678 - auc_34: 0.8012\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4910 - accuracy: 0.7397 - mae: 0.3331 - mse: 0.1664 - auc_34: 0.8050\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4898 - accuracy: 0.7410 - mae: 0.3318 - mse: 0.1658 - auc_34: 0.8066\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4888 - accuracy: 0.7408 - mae: 0.3315 - mse: 0.1656 - auc_34: 0.8071\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4853 - accuracy: 0.7422 - mae: 0.3289 - mse: 0.1643 - auc_34: 0.8105\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4832 - accuracy: 0.7437 - mae: 0.3272 - mse: 0.1634 - auc_34: 0.8127\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4806 - accuracy: 0.7453 - mae: 0.3255 - mse: 0.1626 - auc_34: 0.8149\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4789 - accuracy: 0.7463 - mae: 0.3242 - mse: 0.1619 - auc_34: 0.8163\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4763 - accuracy: 0.7477 - mae: 0.3223 - mse: 0.1610 - auc_34: 0.8189\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "72\n",
      "72\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4796 - accuracy: 0.7465 - mae: 0.3246 - mse: 0.1622 - auc_34: 0.8160\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4781 - accuracy: 0.7471 - mae: 0.3233 - mse: 0.1616 - auc_34: 0.8174\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4756 - accuracy: 0.7482 - mae: 0.3218 - mse: 0.1608 - auc_34: 0.8194\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4748 - accuracy: 0.7483 - mae: 0.3213 - mse: 0.1605 - auc_34: 0.8199\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4699 - accuracy: 0.7512 - mae: 0.3177 - mse: 0.1588 - auc_34: 0.8244\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4692 - accuracy: 0.7511 - mae: 0.3172 - mse: 0.1585 - auc_34: 0.8248\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4674 - accuracy: 0.7518 - mae: 0.3162 - mse: 0.1580 - auc_34: 0.8262\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4671 - accuracy: 0.7523 - mae: 0.3155 - mse: 0.1576 - auc_34: 0.8269\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4647 - accuracy: 0.7541 - mae: 0.3139 - mse: 0.1569 - auc_34: 0.8288\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4657 - accuracy: 0.7530 - mae: 0.3147 - mse: 0.1573 - auc_34: 0.8279\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4643 - accuracy: 0.7545 - mae: 0.3135 - mse: 0.1567 - auc_34: 0.8293\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4619 - accuracy: 0.7546 - mae: 0.3118 - mse: 0.1559 - auc_34: 0.8311\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4628 - accuracy: 0.7536 - mae: 0.3127 - mse: 0.1562 - auc_34: 0.8301\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4601 - accuracy: 0.7560 - mae: 0.3107 - mse: 0.1554 - auc_34: 0.8324\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4605 - accuracy: 0.7555 - mae: 0.3110 - mse: 0.1554 - auc_34: 0.8322\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4575 - accuracy: 0.7577 - mae: 0.3089 - mse: 0.1543 - auc_34: 0.8350\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4586 - accuracy: 0.7575 - mae: 0.3094 - mse: 0.1546 - auc_34: 0.8341\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4551 - accuracy: 0.7599 - mae: 0.3066 - mse: 0.1533 - auc_34: 0.8371\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4569 - accuracy: 0.7578 - mae: 0.3080 - mse: 0.1540 - auc_34: 0.8354\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4534 - accuracy: 0.7600 - mae: 0.3056 - mse: 0.1527 - auc_34: 0.8384\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4558 - accuracy: 0.7593 - mae: 0.3072 - mse: 0.1535 - auc_34: 0.8366\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4539 - accuracy: 0.7596 - mae: 0.3059 - mse: 0.1529 - auc_34: 0.8377\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4519 - accuracy: 0.7619 - mae: 0.3044 - mse: 0.1521 - auc_34: 0.8398\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4513 - accuracy: 0.7624 - mae: 0.3037 - mse: 0.1517 - auc_34: 0.8405\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4486 - accuracy: 0.7619 - mae: 0.3023 - mse: 0.1511 - auc_34: 0.8419\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4489 - accuracy: 0.7631 - mae: 0.3023 - mse: 0.1511 - auc_34: 0.8422\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4486 - accuracy: 0.7628 - mae: 0.3020 - mse: 0.1510 - auc_34: 0.8424\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4471 - accuracy: 0.7646 - mae: 0.3009 - mse: 0.1504 - auc_34: 0.8437\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4481 - accuracy: 0.7639 - mae: 0.3017 - mse: 0.1509 - auc_34: 0.8428\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4479 - accuracy: 0.7647 - mae: 0.3013 - mse: 0.1506 - auc_34: 0.8432\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4477 - accuracy: 0.7636 - mae: 0.3011 - mse: 0.1505 - auc_34: 0.8433\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4460 - accuracy: 0.7649 - mae: 0.3002 - mse: 0.1501 - auc_34: 0.8445\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4459 - accuracy: 0.7646 - mae: 0.3000 - mse: 0.1500 - auc_34: 0.8445\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4452 - accuracy: 0.7658 - mae: 0.2995 - mse: 0.1497 - auc_34: 0.8451\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4456 - accuracy: 0.7652 - mae: 0.2997 - mse: 0.1498 - auc_34: 0.8451\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4433 - accuracy: 0.7671 - mae: 0.2980 - mse: 0.1489 - auc_34: 0.8470\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4421 - accuracy: 0.7673 - mae: 0.2973 - mse: 0.1486 - auc_34: 0.8477\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4399 - accuracy: 0.7680 - mae: 0.2956 - mse: 0.1478 - auc_34: 0.8493\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4402 - accuracy: 0.7685 - mae: 0.2959 - mse: 0.1479 - auc_34: 0.8491\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4427 - accuracy: 0.7674 - mae: 0.2977 - mse: 0.1488 - auc_34: 0.8472\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4398 - accuracy: 0.7679 - mae: 0.2952 - mse: 0.1476 - auc_34: 0.8497\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4384 - accuracy: 0.7697 - mae: 0.2943 - mse: 0.1471 - auc_34: 0.8508\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4388 - accuracy: 0.7700 - mae: 0.2946 - mse: 0.1473 - auc_34: 0.8505\n",
      "Epoch 14/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4370 - accuracy: 0.7696 - mae: 0.2937 - mse: 0.1468 - auc_34: 0.8515\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4390 - accuracy: 0.7686 - mae: 0.2948 - mse: 0.1475 - auc_34: 0.8500\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "5\n",
      "781/781 [==============================] - 1s 663us/step - loss: 0.3021 - accuracy: 0.8927 - mae: 0.2244 - mse: 0.0936 - auc_34: 0.7094\n",
      "train positive label: 440 - train negative label: 224458\n",
      "up and down sampling => train positive label: 112640 - train negative label: 224458\n",
      "Test positive label: 49 - Test negative label: 24939\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.6093 - accuracy: 0.6740 - mae: 0.4201 - mse: 0.2105 - auc_35: 0.6398\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5802 - accuracy: 0.6951 - mae: 0.3976 - mse: 0.1986 - auc_35: 0.6973\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5708 - accuracy: 0.7018 - mae: 0.3903 - mse: 0.1949 - auc_35: 0.7117\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5615 - accuracy: 0.7060 - mae: 0.3834 - mse: 0.1915 - auc_35: 0.7244\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5516 - accuracy: 0.7100 - mae: 0.3764 - mse: 0.1880 - auc_35: 0.7365\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5427 - accuracy: 0.7143 - mae: 0.3705 - mse: 0.1851 - auc_35: 0.7473\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5328 - accuracy: 0.7189 - mae: 0.3637 - mse: 0.1816 - auc_35: 0.7595\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5265 - accuracy: 0.7230 - mae: 0.3588 - mse: 0.1792 - auc_35: 0.7674\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5238 - accuracy: 0.7245 - mae: 0.3571 - mse: 0.1783 - auc_35: 0.7698\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5197 - accuracy: 0.7246 - mae: 0.3544 - mse: 0.1771 - auc_35: 0.7742\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5133 - accuracy: 0.7273 - mae: 0.3499 - mse: 0.1748 - auc_35: 0.7813\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5072 - accuracy: 0.7314 - mae: 0.3454 - mse: 0.1725 - auc_35: 0.7883\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.5004 - accuracy: 0.7351 - mae: 0.3403 - mse: 0.1700 - auc_35: 0.7954\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4944 - accuracy: 0.7386 - mae: 0.3358 - mse: 0.1678 - auc_35: 0.8016\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4901 - accuracy: 0.7413 - mae: 0.3327 - mse: 0.1661 - auc_35: 0.8063\n",
      "73\n",
      "73\n",
      "73\n",
      "72\n",
      "72\n",
      "72\n",
      "72\n",
      "71\n",
      "71\n",
      "70\n",
      "70\n",
      "70\n",
      "69\n",
      "67\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4842 - accuracy: 0.7456 - mae: 0.3284 - mse: 0.1640 - auc_35: 0.8118\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4811 - accuracy: 0.7469 - mae: 0.3259 - mse: 0.1628 - auc_35: 0.8149\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4752 - accuracy: 0.7514 - mae: 0.3215 - mse: 0.1606 - auc_35: 0.8206\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4717 - accuracy: 0.7532 - mae: 0.3191 - mse: 0.1594 - auc_35: 0.8233\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4688 - accuracy: 0.7543 - mae: 0.3168 - mse: 0.1582 - auc_35: 0.8263\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4648 - accuracy: 0.7577 - mae: 0.3136 - mse: 0.1567 - auc_35: 0.8300\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4629 - accuracy: 0.7577 - mae: 0.3126 - mse: 0.1562 - auc_35: 0.8311\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4573 - accuracy: 0.7613 - mae: 0.3083 - mse: 0.1540 - auc_35: 0.8362\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4580 - accuracy: 0.7618 - mae: 0.3085 - mse: 0.1542 - auc_35: 0.8358\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4542 - accuracy: 0.7634 - mae: 0.3060 - mse: 0.1529 - auc_35: 0.8388\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4510 - accuracy: 0.7660 - mae: 0.3034 - mse: 0.1516 - auc_35: 0.8417\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4532 - accuracy: 0.7633 - mae: 0.3054 - mse: 0.1526 - auc_35: 0.8393\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4527 - accuracy: 0.7647 - mae: 0.3048 - mse: 0.1523 - auc_35: 0.8400\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4529 - accuracy: 0.7634 - mae: 0.3052 - mse: 0.1525 - auc_35: 0.8396\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4502 - accuracy: 0.7649 - mae: 0.3031 - mse: 0.1515 - auc_35: 0.8417\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "74\n",
      "74\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4478 - accuracy: 0.7671 - mae: 0.3012 - mse: 0.1505 - auc_35: 0.8440\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4471 - accuracy: 0.7678 - mae: 0.3005 - mse: 0.1502 - auc_35: 0.8446\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4442 - accuracy: 0.7699 - mae: 0.2985 - mse: 0.1492 - auc_35: 0.8469\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4418 - accuracy: 0.7714 - mae: 0.2968 - mse: 0.1483 - auc_35: 0.8489\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4414 - accuracy: 0.7718 - mae: 0.2961 - mse: 0.1481 - auc_35: 0.8494\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4396 - accuracy: 0.7731 - mae: 0.2950 - mse: 0.1474 - auc_35: 0.8507\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4379 - accuracy: 0.7744 - mae: 0.2937 - mse: 0.1467 - auc_35: 0.8522\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4389 - accuracy: 0.7732 - mae: 0.2946 - mse: 0.1472 - auc_35: 0.8512\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4347 - accuracy: 0.7760 - mae: 0.2916 - mse: 0.1456 - auc_35: 0.8545\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4350 - accuracy: 0.7760 - mae: 0.2915 - mse: 0.1457 - auc_35: 0.8544\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4335 - accuracy: 0.7763 - mae: 0.2905 - mse: 0.1452 - auc_35: 0.8554\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4317 - accuracy: 0.7783 - mae: 0.2891 - mse: 0.1445 - auc_35: 0.8570\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4305 - accuracy: 0.7792 - mae: 0.2884 - mse: 0.1441 - auc_35: 0.8578\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4293 - accuracy: 0.7797 - mae: 0.2873 - mse: 0.1436 - auc_35: 0.8587\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4257 - accuracy: 0.7828 - mae: 0.2846 - mse: 0.1422 - auc_35: 0.8616\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "76\n",
      "76\n",
      "76\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4258 - accuracy: 0.7820 - mae: 0.2846 - mse: 0.1423 - auc_35: 0.8614\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4256 - accuracy: 0.7820 - mae: 0.2847 - mse: 0.1423 - auc_35: 0.8615\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4258 - accuracy: 0.7829 - mae: 0.2847 - mse: 0.1422 - auc_35: 0.8616\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4225 - accuracy: 0.7845 - mae: 0.2825 - mse: 0.1412 - auc_35: 0.8638\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4233 - accuracy: 0.7841 - mae: 0.2828 - mse: 0.1413 - auc_35: 0.8633\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4217 - accuracy: 0.7846 - mae: 0.2816 - mse: 0.1408 - auc_35: 0.8646\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4186 - accuracy: 0.7872 - mae: 0.2794 - mse: 0.1396 - auc_35: 0.8669\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4202 - accuracy: 0.7855 - mae: 0.2808 - mse: 0.1403 - auc_35: 0.8655\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4204 - accuracy: 0.7858 - mae: 0.2806 - mse: 0.1402 - auc_35: 0.8656\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4188 - accuracy: 0.7876 - mae: 0.2791 - mse: 0.1395 - auc_35: 0.8671\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4157 - accuracy: 0.7892 - mae: 0.2774 - mse: 0.1386 - auc_35: 0.8689\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4170 - accuracy: 0.7882 - mae: 0.2781 - mse: 0.1390 - auc_35: 0.8680\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4154 - accuracy: 0.7893 - mae: 0.2771 - mse: 0.1385 - auc_35: 0.8691\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4146 - accuracy: 0.7900 - mae: 0.2762 - mse: 0.1381 - auc_35: 0.8699\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4145 - accuracy: 0.7896 - mae: 0.2763 - mse: 0.1381 - auc_35: 0.8699\n",
      "79\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4130 - accuracy: 0.7907 - mae: 0.2751 - mse: 0.1375 - auc_35: 0.8711\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4107 - accuracy: 0.7918 - mae: 0.2735 - mse: 0.1367 - auc_35: 0.8724\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4118 - accuracy: 0.7909 - mae: 0.2742 - mse: 0.1370 - auc_35: 0.8718\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4113 - accuracy: 0.7908 - mae: 0.2739 - mse: 0.1370 - auc_35: 0.8720\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4095 - accuracy: 0.7933 - mae: 0.2725 - mse: 0.1362 - auc_35: 0.8735\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4107 - accuracy: 0.7923 - mae: 0.2734 - mse: 0.1366 - auc_35: 0.8728\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4076 - accuracy: 0.7946 - mae: 0.2710 - mse: 0.1354 - auc_35: 0.8748\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4089 - accuracy: 0.7933 - mae: 0.2722 - mse: 0.1361 - auc_35: 0.8738\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4045 - accuracy: 0.7966 - mae: 0.2689 - mse: 0.1343 - auc_35: 0.8771\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4047 - accuracy: 0.7963 - mae: 0.2690 - mse: 0.1345 - auc_35: 0.8767\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4031 - accuracy: 0.7971 - mae: 0.2678 - mse: 0.1339 - auc_35: 0.8779\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4073 - accuracy: 0.7944 - mae: 0.2708 - mse: 0.1353 - auc_35: 0.8752\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4028 - accuracy: 0.7968 - mae: 0.2677 - mse: 0.1338 - auc_35: 0.8780\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4030 - accuracy: 0.7959 - mae: 0.2681 - mse: 0.1340 - auc_35: 0.8776\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4006 - accuracy: 0.7978 - mae: 0.2664 - mse: 0.1331 - auc_35: 0.8793\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "5\n",
      "781/781 [==============================] - 0s 595us/step - loss: 0.2764 - accuracy: 0.8656 - mae: 0.1979 - mse: 0.0870 - auc_35: 0.6582\n",
      "train positive label: 435 - train negative label: 224463\n",
      "up and down sampling => train positive label: 112665 - train negative label: 224463\n",
      "Test positive label: 54 - Test negative label: 24934\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.6129 - accuracy: 0.6643 - mae: 0.4237 - mse: 0.2123 - auc_36: 0.6309\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5817 - accuracy: 0.6882 - mae: 0.4001 - mse: 0.1997 - auc_36: 0.6944\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5673 - accuracy: 0.7024 - mae: 0.3881 - mse: 0.1937 - auc_36: 0.7174\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5568 - accuracy: 0.7094 - mae: 0.3794 - mse: 0.1895 - auc_36: 0.7320\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5504 - accuracy: 0.7141 - mae: 0.3745 - mse: 0.1871 - auc_36: 0.7412\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5407 - accuracy: 0.7194 - mae: 0.3680 - mse: 0.1838 - auc_36: 0.7521\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5354 - accuracy: 0.7219 - mae: 0.3643 - mse: 0.1820 - auc_36: 0.7583\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5293 - accuracy: 0.7259 - mae: 0.3600 - mse: 0.1798 - auc_36: 0.7651\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5263 - accuracy: 0.7279 - mae: 0.3577 - mse: 0.1787 - auc_36: 0.7682\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5258 - accuracy: 0.7282 - mae: 0.3576 - mse: 0.1787 - auc_36: 0.7681\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5226 - accuracy: 0.7288 - mae: 0.3556 - mse: 0.1777 - auc_36: 0.7708\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5175 - accuracy: 0.7304 - mae: 0.3523 - mse: 0.1760 - auc_36: 0.7760\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5129 - accuracy: 0.7327 - mae: 0.3490 - mse: 0.1744 - auc_36: 0.7805\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5090 - accuracy: 0.7331 - mae: 0.3469 - mse: 0.1733 - auc_36: 0.7839\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5047 - accuracy: 0.7351 - mae: 0.3437 - mse: 0.1718 - auc_36: 0.7889\n",
      "73\n",
      "73\n",
      "73\n",
      "72\n",
      "72\n",
      "72\n",
      "72\n",
      "72\n",
      "71\n",
      "71\n",
      "70\n",
      "70\n",
      "68\n",
      "66\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.5011 - accuracy: 0.7365 - mae: 0.3411 - mse: 0.1704 - auc_36: 0.7929\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.4985 - accuracy: 0.7378 - mae: 0.3392 - mse: 0.1695 - auc_36: 0.7957\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4953 - accuracy: 0.7395 - mae: 0.3366 - mse: 0.1682 - auc_36: 0.7992\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 9s 4ms/step - loss: 0.4919 - accuracy: 0.7391 - mae: 0.3347 - mse: 0.1673 - auc_36: 0.8019\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4899 - accuracy: 0.7412 - mae: 0.3331 - mse: 0.1665 - auc_36: 0.8041\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4872 - accuracy: 0.7423 - mae: 0.3312 - mse: 0.1655 - auc_36: 0.8070\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4848 - accuracy: 0.7439 - mae: 0.3292 - mse: 0.1646 - auc_36: 0.8094\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4820 - accuracy: 0.7457 - mae: 0.3274 - mse: 0.1636 - auc_36: 0.8121\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4799 - accuracy: 0.7454 - mae: 0.3262 - mse: 0.1629 - auc_36: 0.8138\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4768 - accuracy: 0.7479 - mae: 0.3235 - mse: 0.1617 - auc_36: 0.8171\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4742 - accuracy: 0.7489 - mae: 0.3220 - mse: 0.1609 - auc_36: 0.8190\n",
      "Epoch 12/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4734 - accuracy: 0.7493 - mae: 0.3212 - mse: 0.1606 - auc_36: 0.8201\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4721 - accuracy: 0.7505 - mae: 0.3202 - mse: 0.1600 - auc_36: 0.8214\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4703 - accuracy: 0.7516 - mae: 0.3189 - mse: 0.1594 - auc_36: 0.8230\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4676 - accuracy: 0.7531 - mae: 0.3171 - mse: 0.1584 - auc_36: 0.8253\n",
      "75\n",
      "75\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "74\n",
      "73\n",
      "73\n",
      "73\n",
      "73\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4679 - accuracy: 0.7519 - mae: 0.3171 - mse: 0.1586 - auc_36: 0.8250\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4642 - accuracy: 0.7552 - mae: 0.3146 - mse: 0.1572 - auc_36: 0.8285\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4633 - accuracy: 0.7555 - mae: 0.3138 - mse: 0.1568 - auc_36: 0.8294\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4600 - accuracy: 0.7581 - mae: 0.3111 - mse: 0.1555 - auc_36: 0.8325\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4598 - accuracy: 0.7580 - mae: 0.3108 - mse: 0.1554 - auc_36: 0.8327\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4576 - accuracy: 0.7589 - mae: 0.3097 - mse: 0.1548 - auc_36: 0.8343\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4570 - accuracy: 0.7604 - mae: 0.3089 - mse: 0.1544 - auc_36: 0.8352\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4555 - accuracy: 0.7611 - mae: 0.3078 - mse: 0.1539 - auc_36: 0.8364\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4546 - accuracy: 0.7616 - mae: 0.3071 - mse: 0.1535 - auc_36: 0.8371\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4521 - accuracy: 0.7634 - mae: 0.3052 - mse: 0.1525 - auc_36: 0.8396\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 11s 4ms/step - loss: 0.4491 - accuracy: 0.7650 - mae: 0.3032 - mse: 0.1515 - auc_36: 0.8418\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4497 - accuracy: 0.7645 - mae: 0.3035 - mse: 0.1517 - auc_36: 0.8415\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4467 - accuracy: 0.7669 - mae: 0.3011 - mse: 0.1505 - auc_36: 0.8441\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4463 - accuracy: 0.7671 - mae: 0.3009 - mse: 0.1505 - auc_36: 0.8440\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4446 - accuracy: 0.7684 - mae: 0.2995 - mse: 0.1497 - auc_36: 0.8460\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "76\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "75\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4456 - accuracy: 0.7676 - mae: 0.3003 - mse: 0.1501 - auc_36: 0.8450\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4412 - accuracy: 0.7704 - mae: 0.2969 - mse: 0.1485 - auc_36: 0.8486\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4417 - accuracy: 0.7695 - mae: 0.2975 - mse: 0.1487 - auc_36: 0.8481\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4418 - accuracy: 0.7697 - mae: 0.2975 - mse: 0.1487 - auc_36: 0.8481\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4396 - accuracy: 0.7707 - mae: 0.2960 - mse: 0.1479 - auc_36: 0.8498\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4370 - accuracy: 0.7725 - mae: 0.2940 - mse: 0.1470 - auc_36: 0.8519\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4383 - accuracy: 0.7714 - mae: 0.2950 - mse: 0.1475 - auc_36: 0.8507\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4353 - accuracy: 0.7736 - mae: 0.2928 - mse: 0.1463 - auc_36: 0.8532\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4350 - accuracy: 0.7732 - mae: 0.2924 - mse: 0.1462 - auc_36: 0.8535\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4333 - accuracy: 0.7750 - mae: 0.2911 - mse: 0.1455 - auc_36: 0.8551\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4327 - accuracy: 0.7755 - mae: 0.2907 - mse: 0.1454 - auc_36: 0.8553\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4310 - accuracy: 0.7764 - mae: 0.2894 - mse: 0.1447 - auc_36: 0.8566\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4298 - accuracy: 0.7780 - mae: 0.2883 - mse: 0.1441 - auc_36: 0.8579\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4318 - accuracy: 0.7757 - mae: 0.2899 - mse: 0.1450 - auc_36: 0.8560\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4293 - accuracy: 0.7778 - mae: 0.2882 - mse: 0.1440 - auc_36: 0.8582\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "76\n",
      "76\n",
      "77\n",
      "76\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4297 - accuracy: 0.7782 - mae: 0.2883 - mse: 0.1441 - auc_36: 0.8579\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4276 - accuracy: 0.7777 - mae: 0.2868 - mse: 0.1434 - auc_36: 0.8592\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4276 - accuracy: 0.7784 - mae: 0.2869 - mse: 0.1434 - auc_36: 0.8594\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4295 - accuracy: 0.7778 - mae: 0.2881 - mse: 0.1441 - auc_36: 0.8580\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4297 - accuracy: 0.7774 - mae: 0.2879 - mse: 0.1440 - auc_36: 0.8581\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4319 - accuracy: 0.7770 - mae: 0.2896 - mse: 0.1448 - auc_36: 0.8566\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4276 - accuracy: 0.7783 - mae: 0.2867 - mse: 0.1433 - auc_36: 0.8595\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4261 - accuracy: 0.7798 - mae: 0.2858 - mse: 0.1429 - auc_36: 0.8605\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4277 - accuracy: 0.7787 - mae: 0.2866 - mse: 0.1433 - auc_36: 0.8596\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4261 - accuracy: 0.7800 - mae: 0.2857 - mse: 0.1427 - auc_36: 0.8608\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4231 - accuracy: 0.7813 - mae: 0.2834 - mse: 0.1417 - auc_36: 0.8629\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4207 - accuracy: 0.7823 - mae: 0.2817 - mse: 0.1409 - auc_36: 0.8645\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4191 - accuracy: 0.7840 - mae: 0.2804 - mse: 0.1401 - auc_36: 0.8660\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4204 - accuracy: 0.7825 - mae: 0.2815 - mse: 0.1408 - auc_36: 0.8648\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4183 - accuracy: 0.7842 - mae: 0.2798 - mse: 0.1400 - auc_36: 0.8664\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "77\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4198 - accuracy: 0.7843 - mae: 0.2808 - mse: 0.1403 - auc_36: 0.8657\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4199 - accuracy: 0.7834 - mae: 0.2808 - mse: 0.1403 - auc_36: 0.8655\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4161 - accuracy: 0.7863 - mae: 0.2783 - mse: 0.1391 - auc_36: 0.8681\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4153 - accuracy: 0.7859 - mae: 0.2775 - mse: 0.1387 - auc_36: 0.8688\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4147 - accuracy: 0.7870 - mae: 0.2769 - mse: 0.1385 - auc_36: 0.8693\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4123 - accuracy: 0.7887 - mae: 0.2754 - mse: 0.1377 - auc_36: 0.8709\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4155 - accuracy: 0.7864 - mae: 0.2776 - mse: 0.1387 - auc_36: 0.8688\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4167 - accuracy: 0.7847 - mae: 0.2785 - mse: 0.1393 - auc_36: 0.8676\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4186 - accuracy: 0.7849 - mae: 0.2797 - mse: 0.1398 - auc_36: 0.8666\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4134 - accuracy: 0.7877 - mae: 0.2760 - mse: 0.1380 - auc_36: 0.8702\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4127 - accuracy: 0.7882 - mae: 0.2758 - mse: 0.1378 - auc_36: 0.8706\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4112 - accuracy: 0.7890 - mae: 0.2747 - mse: 0.1372 - auc_36: 0.8717\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4126 - accuracy: 0.7880 - mae: 0.2754 - mse: 0.1377 - auc_36: 0.8707\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4123 - accuracy: 0.7883 - mae: 0.2755 - mse: 0.1377 - auc_36: 0.8708\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4078 - accuracy: 0.7913 - mae: 0.2721 - mse: 0.1360 - auc_36: 0.8741\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "78\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4079 - accuracy: 0.7914 - mae: 0.2719 - mse: 0.1359 - auc_36: 0.8742\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4098 - accuracy: 0.7903 - mae: 0.2731 - mse: 0.1365 - auc_36: 0.8731\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4107 - accuracy: 0.7905 - mae: 0.2734 - mse: 0.1367 - auc_36: 0.8726\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4090 - accuracy: 0.7909 - mae: 0.2729 - mse: 0.1365 - auc_36: 0.8733\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4070 - accuracy: 0.7917 - mae: 0.2714 - mse: 0.1357 - auc_36: 0.8747\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4077 - accuracy: 0.7920 - mae: 0.2716 - mse: 0.1358 - auc_36: 0.8745\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4075 - accuracy: 0.7922 - mae: 0.2715 - mse: 0.1358 - auc_36: 0.8745\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4044 - accuracy: 0.7943 - mae: 0.2692 - mse: 0.1345 - auc_36: 0.8768\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4033 - accuracy: 0.7943 - mae: 0.2685 - mse: 0.1342 - auc_36: 0.8773\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4044 - accuracy: 0.7945 - mae: 0.2689 - mse: 0.1344 - auc_36: 0.8770\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4124 - accuracy: 0.7899 - mae: 0.2751 - mse: 0.1375 - auc_36: 0.8713\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4097 - accuracy: 0.7906 - mae: 0.2730 - mse: 0.1365 - auc_36: 0.8731\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4055 - accuracy: 0.7932 - mae: 0.2700 - mse: 0.1349 - auc_36: 0.8762\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4079 - accuracy: 0.7923 - mae: 0.2714 - mse: 0.1357 - auc_36: 0.8745\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4031 - accuracy: 0.7954 - mae: 0.2683 - mse: 0.1340 - auc_36: 0.8778\n",
      "79\n",
      "79\n",
      "79\n",
      "78\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4047 - accuracy: 0.7944 - mae: 0.2693 - mse: 0.1347 - auc_36: 0.8766\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4007 - accuracy: 0.7963 - mae: 0.2663 - mse: 0.1331 - auc_36: 0.8794\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4022 - accuracy: 0.7953 - mae: 0.2675 - mse: 0.1337 - auc_36: 0.8782\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4010 - accuracy: 0.7970 - mae: 0.2666 - mse: 0.1333 - auc_36: 0.8792\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3999 - accuracy: 0.7983 - mae: 0.2657 - mse: 0.1328 - auc_36: 0.8801\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4005 - accuracy: 0.7964 - mae: 0.2664 - mse: 0.1332 - auc_36: 0.8793\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3972 - accuracy: 0.7983 - mae: 0.2638 - mse: 0.1319 - auc_36: 0.8816\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4007 - accuracy: 0.7970 - mae: 0.2662 - mse: 0.1330 - auc_36: 0.8795\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3997 - accuracy: 0.7977 - mae: 0.2653 - mse: 0.1327 - auc_36: 0.8801\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4011 - accuracy: 0.7965 - mae: 0.2664 - mse: 0.1332 - auc_36: 0.8792\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3971 - accuracy: 0.7988 - mae: 0.2640 - mse: 0.1319 - auc_36: 0.8815\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3991 - accuracy: 0.7977 - mae: 0.2649 - mse: 0.1325 - auc_36: 0.8806\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3990 - accuracy: 0.7979 - mae: 0.2649 - mse: 0.1325 - auc_36: 0.8805\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3965 - accuracy: 0.7985 - mae: 0.2636 - mse: 0.1317 - auc_36: 0.8820\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3944 - accuracy: 0.8005 - mae: 0.2618 - mse: 0.1308 - auc_36: 0.8835\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3982 - accuracy: 0.7979 - mae: 0.2645 - mse: 0.1322 - auc_36: 0.8810\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3957 - accuracy: 0.8000 - mae: 0.2626 - mse: 0.1312 - auc_36: 0.8828\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3957 - accuracy: 0.8002 - mae: 0.2624 - mse: 0.1312 - auc_36: 0.8827\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3966 - accuracy: 0.7989 - mae: 0.2634 - mse: 0.1316 - auc_36: 0.8821\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3976 - accuracy: 0.7990 - mae: 0.2637 - mse: 0.1319 - auc_36: 0.8816\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4015 - accuracy: 0.7968 - mae: 0.2666 - mse: 0.1333 - auc_36: 0.8790\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4005 - accuracy: 0.7966 - mae: 0.2664 - mse: 0.1331 - auc_36: 0.8793\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3985 - accuracy: 0.7985 - mae: 0.2649 - mse: 0.1324 - auc_36: 0.8807\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.4000 - accuracy: 0.7973 - mae: 0.2656 - mse: 0.1327 - auc_36: 0.8799\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3987 - accuracy: 0.7978 - mae: 0.2647 - mse: 0.1323 - auc_36: 0.8808\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3947 - accuracy: 0.8009 - mae: 0.2617 - mse: 0.1309 - auc_36: 0.8833\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3930 - accuracy: 0.8014 - mae: 0.2609 - mse: 0.1304 - auc_36: 0.8843\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3945 - accuracy: 0.8008 - mae: 0.2617 - mse: 0.1308 - auc_36: 0.8834\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3942 - accuracy: 0.8011 - mae: 0.2612 - mse: 0.1306 - auc_36: 0.8839\n",
      "Epoch 15/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3959 - accuracy: 0.8015 - mae: 0.2626 - mse: 0.1311 - auc_36: 0.8829\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "79\n",
      "80\n",
      "80\n",
      "79\n",
      "Epoch 1/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3939 - accuracy: 0.8017 - mae: 0.2610 - mse: 0.1305 - auc_36: 0.8840\n",
      "Epoch 2/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3910 - accuracy: 0.8026 - mae: 0.2591 - mse: 0.1296 - auc_36: 0.8857\n",
      "Epoch 3/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3913 - accuracy: 0.8031 - mae: 0.2593 - mse: 0.1295 - auc_36: 0.8858\n",
      "Epoch 4/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3921 - accuracy: 0.8030 - mae: 0.2598 - mse: 0.1299 - auc_36: 0.8851\n",
      "Epoch 5/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3920 - accuracy: 0.8027 - mae: 0.2596 - mse: 0.1297 - auc_36: 0.8855\n",
      "Epoch 6/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3911 - accuracy: 0.8034 - mae: 0.2588 - mse: 0.1294 - auc_36: 0.8859\n",
      "Epoch 7/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3894 - accuracy: 0.8036 - mae: 0.2579 - mse: 0.1289 - auc_36: 0.8867\n",
      "Epoch 8/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3903 - accuracy: 0.8036 - mae: 0.2584 - mse: 0.1292 - auc_36: 0.8863\n",
      "Epoch 9/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3910 - accuracy: 0.8037 - mae: 0.2588 - mse: 0.1294 - auc_36: 0.8860\n",
      "Epoch 10/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3890 - accuracy: 0.8031 - mae: 0.2578 - mse: 0.1288 - auc_36: 0.8870\n",
      "Epoch 11/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3893 - accuracy: 0.8031 - mae: 0.2578 - mse: 0.1289 - auc_36: 0.8868\n",
      "Epoch 12/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3870 - accuracy: 0.8047 - mae: 0.2563 - mse: 0.1281 - auc_36: 0.8882\n",
      "Epoch 13/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3923 - accuracy: 0.8033 - mae: 0.2598 - mse: 0.1298 - auc_36: 0.8853\n",
      "Epoch 14/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3910 - accuracy: 0.8039 - mae: 0.2587 - mse: 0.1294 - auc_36: 0.8860\n",
      "Epoch 15/15\n",
      "2634/2634 [==============================] - 10s 4ms/step - loss: 0.3891 - accuracy: 0.8045 - mae: 0.2574 - mse: 0.1287 - auc_36: 0.8871\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "80\n",
      "10\n",
      "781/781 [==============================] - 1s 707us/step - loss: 0.2747 - accuracy: 0.8484 - mae: 0.1962 - mse: 0.0882 - auc_36: 0.6405\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "Epoch_S = 15\n",
    "\n",
    "def evaluate_model(df, k = 10 , shuffle = False):\n",
    "    result =[]    \n",
    "\n",
    "    kf = KFold(n_splits=10, shuffle= shuffle, random_state=None)\n",
    "    \n",
    "    for train_index, test_index in kf.split(df):\n",
    "\n",
    "        train_ds = [df[index] for index in train_index] \n",
    "        \n",
    "        valid_ds = [df[index] for index in test_index]\n",
    "        \n",
    "        label_pos , label_neg, _ , _ = count_lablel(train_ds)\n",
    "        print(f'train positive label: {label_pos} - train negative label: {label_neg}')\n",
    "        \n",
    "        train_ds = up_and_down_Samplenig(train_ds, sacale_upsampling = 2 , scale_downsampling = 0.5)\n",
    "        \n",
    "        label_pos , label_neg , _ , _ = count_lablel(train_ds)\n",
    "        print(f'up and down sampling => train positive label: {label_pos} - train negative label: {label_neg}')\n",
    "\n",
    "        label_pos , label_neg, _ , _ = count_lablel(valid_ds)\n",
    "        print(f'Test positive label: {label_pos} - Test negative label: {label_neg}')\n",
    "\n",
    "        l_train = []\n",
    "        r_train = []\n",
    "        lbls_train = []\n",
    "        l_valid = []\n",
    "        r_valid = []\n",
    "        lbls_valid = []\n",
    "\n",
    "        for i , data in enumerate(train_ds):\n",
    "            embbed_drug, onehot_task, embbed_task, lbl, task_name = data\n",
    "            l_train.append(embbed_drug[0])\n",
    "            r_train.append(embbed_task)\n",
    "            lbls_train.append(lbl.tolist())\n",
    "        \n",
    "        for i , data in enumerate(valid_ds):\n",
    "            embbed_drug, onehot_task, embbed_task, lbl, task_name = data\n",
    "            l_valid.append(embbed_drug[0])\n",
    "            r_valid.append(embbed_task)\n",
    "            lbls_valid.append(lbl.tolist())\n",
    "\n",
    "        l_train = np.array(l_train).reshape(-1,128,1)\n",
    "        r_train = np.array(r_train).reshape(-1,512,1)\n",
    "        lbls_train = np.array(lbls_train)\n",
    "\n",
    "        l_valid = np.array(l_valid).reshape(-1,128,1)\n",
    "        r_valid = np.array(r_valid).reshape(-1,512,1)\n",
    "        lbls_valid = np.array(lbls_valid)\n",
    "\n",
    "        # create neural network model\n",
    "        siamese_net = siamese_model_Canonical_muv()\n",
    "        history = History()\n",
    "        P = siamese_net.fit([l_train, r_train], lbls_train, epochs = Epoch_S, batch_size = 128, callbacks=[history])\n",
    "\n",
    "        for j in range(100):\n",
    "            C=1\n",
    "            Before = int(P.history['accuracy'][-1]*100)\n",
    "            for i in range(2,Epoch_S+1):\n",
    "                if  int(P.history['accuracy'][-i]*100) == Before:\n",
    "                    C=C+1\n",
    "                else:\n",
    "                    C=1\n",
    "                Before=int(P.history['accuracy'][-i]*100)\n",
    "                print(Before)\n",
    "            if C==Epoch_S:\n",
    "                break\n",
    "            P = siamese_net.fit([l_train, r_train], lbls_train, epochs = Epoch_S, batch_size = 128, callbacks=[history])\n",
    "        print(j+1)\n",
    "        \n",
    "        score  = siamese_net.evaluate([l_valid,r_valid], lbls_valid, verbose=1)\n",
    "        a = (score[1],score[4])\n",
    "        result.append(a)\n",
    "    \n",
    "    return result\n",
    "\n",
    "scores = evaluate_model(data_ds, 10, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true,
    "run_control": {
     "marked": true
    }
   },
   "source": [
    "#### Dropout = 0.2 and downsampling = 0.5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.8201208710670471, 0.7097600698471069),\n",
       " (0.8498939275741577, 0.639973521232605),\n",
       " (0.8861899375915527, 0.6834255456924438),\n",
       " (0.8530153036117554, 0.6460467576980591),\n",
       " (0.865900993347168, 0.7278987169265747),\n",
       " (0.9073192477226257, 0.7258636355400085),\n",
       " (0.8746198415756226, 0.6931874752044678),\n",
       " (0.8926684856414795, 0.709435224533081),\n",
       " (0.8656154870986938, 0.6582207679748535),\n",
       " (0.8484072089195251, 0.6405272483825684)]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy= 0.8663751304149627 AUC= 0.6834338963031769 STD_AUC= 0.03317371928095168\n"
     ]
    }
   ],
   "source": [
    "acc = []\n",
    "auc = []\n",
    "for i in scores:\n",
    "    acc.append(i[0])\n",
    "    auc.append(i[1])\n",
    "\n",
    "print(f'accuracy= {np.mean(acc)} AUC= {np.mean(auc)} STD_AUC= {np.std(auc)}')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "zIKQi__XAcia"
   ],
   "provenance": []
  },
  "gpuClass": "standard",
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
